commit a446cd2f22ac04ff718e616246ac11cc0aa17afa parent 174c7b5e2c811a2fb1c545238cc18ce3267bfa44 Author: JohnMcPMS <johnmcp@microsoft.com> Date: Thu, 30 Jan 2020 14:54:29 -0800 Implement Add and RemoveManifest functions for writing to index (#26) Diffstat:
56 files changed, 1731 insertions(+), 129 deletions(-)
diff --git a/src/AppInstallerCLICore/Core.cpp b/src/AppInstallerCLICore/Core.cpp @@ -18,7 +18,7 @@ namespace AppInstaller::CLI // Enable logging (*all* for now, TODO: add common arguments to allow control of logging) Logging::Log().EnableChannel(Logging::Channel::All); Logging::Log().SetLevel(Logging::Level::Verbose); - Logging::AddDefaultFileLogger(); + Logging::AddFileLogger(); Logging::EnableWilFailureTelemetry(); Logging::Telemetry().LogStartup(); diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj @@ -150,6 +150,7 @@ <ItemGroup> <ClCompile Include="Downloader.cpp" /> <ClCompile Include="InstallFlow.cpp" /> + <ClCompile Include="LanguageUtilities.cpp" /> <ClCompile Include="main.cpp"> <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">NotUsing</PrecompiledHeader> <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">NotUsing</PrecompiledHeader> diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters @@ -50,6 +50,9 @@ <ClCompile Include="InstallFlow.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="LanguageUtilities.cpp"> + <Filter>Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <None Include="PropertySheet.props" /> diff --git a/src/AppInstallerCLITests/InstallFlow.cpp b/src/AppInstallerCLITests/InstallFlow.cpp @@ -1,11 +1,12 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. - #include "pch.h" +#include "TestCommon.h" #include "Manifest/Manifest.h" #include "AppInstallerDownloader.h" #include "Workflows/InstallFlow.h" +using namespace TestCommon; using namespace AppInstaller::Workflow; using namespace AppInstaller::Utility; using namespace AppInstaller::Manifest; @@ -19,7 +20,7 @@ public: protected: void DownloadInstaller() override { - this->m_downloadedInstaller = std::filesystem::current_path().append("AppInstallerTestExeInstaller.exe"); + this->m_downloadedInstaller = TestDataFile("AppInstallerTestExeInstaller.exe"); } }; @@ -29,10 +30,12 @@ TEST_CASE("InstallFlowWithTestManifest", "[InstallFlow]") std::filesystem::remove(installResultPath); - auto manifest = Manifest::CreateFromPath("InstallFlowTest.yml"); + auto manifest = Manifest::CreateFromPath(TestDataFile("InstallFlowTest.yml")); - InstallFlowTest testFlow(manifest, std::cout, std::cin); + std::ostringstream installOutput; + InstallFlowTest testFlow(manifest, installOutput, std::cin); testFlow.Install(); + INFO(installOutput.str()); // Verify Installer is called and parameters are passed in. REQUIRE(std::filesystem::exists(installResultPath)); @@ -49,10 +52,12 @@ TEST_CASE("InstallFlowWithNonApplicableArchitecture", "[InstallFlow]") std::filesystem::remove(installResultPath); - auto manifest = Manifest::CreateFromPath("InstallFlowTest_NoApplicableArchitecture.yml"); + auto manifest = Manifest::CreateFromPath(TestDataFile("InstallFlowTest_NoApplicableArchitecture.yml")); - InstallFlowTest testFlow(manifest, std::cout, std::cin); + std::ostringstream installOutput; + InstallFlowTest testFlow(manifest, installOutput, std::cin); REQUIRE_THROWS_WITH(testFlow.Install(), Catch::Contains("No installer with applicable architecture found.")); + INFO(installOutput.str()); // Verify Installer is called and parameters are passed in. REQUIRE(!std::filesystem::exists(installResultPath)); diff --git a/src/AppInstallerCLITests/LanguageUtilities.cpp b/src/AppInstallerCLITests/LanguageUtilities.cpp @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "TestCommon.h" +#include <AppInstallerLanguageUtilities.h> + +using namespace AppInstaller; + + +TEST_CASE("DestructionToken", "[langutil]") +{ + DestructionToken beginToken = true; + DestructionToken endToken = false; + + REQUIRE(beginToken); + REQUIRE(!endToken); + + endToken = std::move(beginToken); + + REQUIRE(!beginToken); + REQUIRE(endToken); +} diff --git a/src/AppInstallerCLITests/SQLiteIndex.cpp b/src/AppInstallerCLITests/SQLiteIndex.cpp @@ -2,13 +2,30 @@ // Licensed under the MIT License. #include "pch.h" #include "TestCommon.h" +#include <SQLiteWrapper.h> +#include <Manifest/Manifest.h> #include <Microsoft/SQLiteIndex.h> +#include <Microsoft/Schema/1_0/IdTable.h> +#include <Microsoft/Schema/1_0/NameTable.h> +#include <Microsoft/Schema/1_0/MonikerTable.h> +#include <Microsoft/Schema/1_0/VersionTable.h> +#include <Microsoft/Schema/1_0/ChannelTable.h> +#include <Microsoft/Schema/1_0/PathPartTable.h> +#include <Microsoft/Schema/1_0/ManifestTable.h> +#include <Microsoft/Schema/1_0/TagsTable.h> +#include <Microsoft/Schema/1_0/CommandsTable.h> +#include <Microsoft/Schema/1_0/ProtocolsTable.h> +#include <Microsoft/Schema/1_0/ExtensionsTable.h> + +using namespace TestCommon; +using namespace AppInstaller::Manifest; using namespace AppInstaller::Repository::Microsoft; +using namespace AppInstaller::Repository::SQLite; TEST_CASE("SQLiteIndexCreateLatestAndReopen", "[sqliteindex]") { - TestCommon::TempFile tempFile{ "repolibtest_tempdb", ".db" }; + TempFile tempFile{ "repolibtest_tempdb", ".db" }; INFO("Using temporary file named: " << tempFile.GetPath()); Schema::Version versionCreated; @@ -43,3 +60,230 @@ TEST_CASE("SQLiteIndexCreateLatestAndReopen", "[sqliteindex]") REQUIRE(versionRead == versionCreated); } } + +TEST_CASE("SQLiteIndexCreateAndAddManifest", "[sqliteindex]") +{ + TempFile tempFile{ "repolibtest_tempdb", ".db" }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + SQLiteIndex index = SQLiteIndex::CreateNew(tempFile, Schema::Version::Latest()); + + Manifest manifest; + manifest.Id = "test.id"; + manifest.Name = "Test Name"; + manifest.AppMoniker = "testmoniker"; + manifest.Version = "1.0.0"; + manifest.Channel = "test"; + manifest.Tags = { "t1", "t2" }; + manifest.Commands = { "test1", "test2" }; + manifest.Protocols = { "htttest" }; + manifest.FileExtensions = { "tst", "test", "testy" }; + + index.AddManifest(manifest, "test/id/test.id-1.0.0.yml"); +} + +TEST_CASE("SQLiteIndexCreateAndAddManifestFile", "[sqliteindex]") +{ + TempFile tempFile{ "repolibtest_tempdb", ".db" }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + SQLiteIndex index = SQLiteIndex::CreateNew(tempFile, Schema::Version::Latest()); + + TestDataFile manifestFile{ "GoodManifest.yml" }; + std::filesystem::path manifestPath{ "microsoft/msixsdk/microsoft.msixsdk-1.7.32.yml" }; + + index.AddManifest(manifestFile, manifestPath); + + // Attempting to add again should fail + REQUIRE_THROWS_HR(index.AddManifest(manifestFile, manifestPath), HRESULT_FROM_WIN32(ERROR_ALREADY_EXISTS)); +} + +TEST_CASE("SQLiteIndex_RemoveManifestFile_NotPresent", "[sqliteindex]") +{ + SQLiteIndex index = SQLiteIndex::CreateNew(SQLITE_MEMORY_DB_CONNECTION_TARGET, Schema::Version::Latest()); + + TestDataFile manifestFile{ "GoodManifest.yml" }; + std::filesystem::path manifestPath{ "microsoft/msixsdk/microsoft.msixsdk-1.7.32.yml" }; + + REQUIRE_THROWS_HR(index.RemoveManifest(manifestFile, manifestPath), E_NOT_SET); +} + +TEST_CASE("SQLiteIndex_RemoveManifest", "[sqliteindex]") +{ + TempFile tempFile{ "repolibtest_tempdb", ".db" }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + std::string manifest1Path = "test/id/test.id-1.0.0.yml"; + Manifest manifest1; + manifest1.Id = "test.id"; + manifest1.Name = "Test Name"; + manifest1.AppMoniker = "testmoniker"; + manifest1.Version = "1.0.0"; + manifest1.Channel = "test"; + manifest1.Tags = { "t1", "t2" }; + manifest1.Commands = { "test1", "test2" }; + manifest1.Protocols = { "htttest" }; + manifest1.FileExtensions = { "tst", "test", "testy" }; + + std::string manifest2Path = "test/woah/test.id-1.0.0.yml"; + Manifest manifest2; + manifest2.Id = "test.woah"; + manifest2.Name = "Test Name WOAH"; + manifest2.AppMoniker = "testmoniker"; + manifest2.Version = "1.0.0"; + manifest2.Channel = "test"; + manifest2.Tags = { "t1" }; + manifest2.Commands = { "test1", "test2", "test3" }; + manifest2.Protocols = {}; + manifest2.FileExtensions = { "tst", "test", "testy" }; + + { + SQLiteIndex index = SQLiteIndex::CreateNew(tempFile, { 1, 0 }); + + index.AddManifest(manifest1, manifest1Path); + index.AddManifest(manifest2, manifest2Path); + + // Now remove manifest1 + index.RemoveManifest(manifest1, manifest1Path); + } + + { + // Open it directly to directly test table state + Connection connection = Connection::Create(tempFile, Connection::OpenDisposition::ReadWrite); + + REQUIRE(!Schema::V1_0::ManifestTable::IsEmpty(connection)); + REQUIRE(!Schema::V1_0::IdTable::IsEmpty(connection)); + REQUIRE(!Schema::V1_0::NameTable::IsEmpty(connection)); + REQUIRE(!Schema::V1_0::MonikerTable::IsEmpty(connection)); + REQUIRE(!Schema::V1_0::VersionTable::IsEmpty(connection)); + REQUIRE(!Schema::V1_0::ChannelTable::IsEmpty(connection)); + REQUIRE(!Schema::V1_0::PathPartTable::IsEmpty(connection)); + REQUIRE(!Schema::V1_0::TagsTable::IsEmpty(connection)); + REQUIRE(!Schema::V1_0::CommandsTable::IsEmpty(connection)); + // Because manifest2 had no protocols + REQUIRE(Schema::V1_0::ProtocolsTable::IsEmpty(connection)); + REQUIRE(!Schema::V1_0::ExtensionsTable::IsEmpty(connection)); + } + + { + SQLiteIndex index = SQLiteIndex::Open(tempFile, SQLiteIndex::OpenDisposition::ReadWrite); + + // Now remove manifest2 + index.RemoveManifest(manifest2, manifest2Path); + } + + // Open it directly to directly test table state + Connection connection = Connection::Create(tempFile, Connection::OpenDisposition::ReadWrite); + + REQUIRE(Schema::V1_0::ManifestTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::IdTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::NameTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::MonikerTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::VersionTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::ChannelTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::PathPartTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::TagsTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::CommandsTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::ProtocolsTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::ExtensionsTable::IsEmpty(connection)); +} + +TEST_CASE("SQLiteIndex_RemoveManifestFile", "[sqliteindex]") +{ + TempFile tempFile{ "repolibtest_tempdb", ".db" }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + { + SQLiteIndex index = SQLiteIndex::CreateNew(tempFile, { 1, 0 }); + + TestDataFile manifestFile{ "GoodManifest.yml" }; + std::filesystem::path manifestPath{ "microsoft/msixsdk/microsoft.msixsdk-1.7.32.yml" }; + + index.AddManifest(manifestFile, manifestPath); + + // Now remove that manifest + index.RemoveManifest(manifestFile, manifestPath); + } + + // Open it directly to directly test table state + Connection connection = Connection::Create(tempFile, Connection::OpenDisposition::ReadWrite); + + REQUIRE(Schema::V1_0::ManifestTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::IdTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::NameTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::MonikerTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::VersionTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::ChannelTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::PathPartTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::TagsTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::CommandsTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::ProtocolsTable::IsEmpty(connection)); + REQUIRE(Schema::V1_0::ExtensionsTable::IsEmpty(connection)); +} + +TEST_CASE("PathPartTable_EnsurePathExists_Negative_Paths", "[sqliteindex][V1_0]") +{ + // Open it directly to directly test pathpart table + Connection connection = Connection::Create(SQLITE_MEMORY_DB_CONNECTION_TARGET, Connection::OpenDisposition::Create); + + REQUIRE_THROWS_HR(Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"()", false), E_INVALIDARG); + REQUIRE_THROWS_HR(Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"(\)", false), E_INVALIDARG); + REQUIRE_THROWS_HR(Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"(/)", false), E_INVALIDARG); + REQUIRE_THROWS_HR(Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"(C:)", false), E_INVALIDARG); + REQUIRE_THROWS_HR(Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"(C:\\)", false), E_INVALIDARG); + REQUIRE_THROWS_HR(Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"(C:\temp\path\file.txt)", false), E_INVALIDARG); + REQUIRE_THROWS_HR(Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"(\temp\path\file.txt)", false), E_INVALIDARG); +} + +TEST_CASE("PathPartTable_EnsurePathExists", "[sqliteindex][V1_0]") +{ + TempFile tempFile{ "repolibtest_tempdb", ".db" }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + // Create the index + { + SQLiteIndex index = SQLiteIndex::CreateNew(tempFile, { 1, 0 }); + Schema::Version versionCreated = index.GetVersion(); + REQUIRE(versionCreated == Schema::Version{ 1, 0 }); + } + + // Open it directly to directly test pathpart table + Connection connection = Connection::Create(tempFile, Connection::OpenDisposition::ReadWrite); + + // attempt to find path that doesn't exist + auto result0 = Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"(a\b\c.txt)", false); + REQUIRE(!std::get<0>(result0)); + + // add path + auto result1 = Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"(a\b\c.txt)", true); + REQUIRE(std::get<0>(result1)); + + // Second time trying to create should return false and same id + auto result2 = Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"(a\b\c.txt)", true); + REQUIRE(!std::get<0>(result2)); + REQUIRE(std::get<1>(result1) == std::get<1>(result2)); + + // Trying to find but not create should return true because it exists + auto result3 = Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"(a\b\c.txt)", false); + REQUIRE(std::get<0>(result3)); + REQUIRE(std::get<1>(result1) == std::get<1>(result3)); + + // attempt to find a different file + auto result4 = Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"(a\b\d.txt)", false); + REQUIRE(!std::get<0>(result4)); + + // add a different file + auto result5 = Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"(a\b\d.txt)", true); + REQUIRE(std::get<0>(result5)); + REQUIRE(std::get<1>(result1) != std::get<1>(result5)); + + // add the same file but deeper + auto result6 = Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"(a\b\d\c.txt)", true); + REQUIRE(std::get<0>(result6)); + REQUIRE(std::get<1>(result1) != std::get<1>(result6)); + + // get the deeper file with extra separators + auto result7 = Schema::V1_0::PathPartTable::EnsurePathExists(connection, R"(a\\b\d\\c.txt)", true); + REQUIRE(!std::get<0>(result7)); + REQUIRE(std::get<1>(result6) == std::get<1>(result7)); +} diff --git a/src/AppInstallerCLITests/SQLiteWrapper.cpp b/src/AppInstallerCLITests/SQLiteWrapper.cpp @@ -6,8 +6,6 @@ using namespace AppInstaller::Repository::SQLite; -#define SQLITE_MEMORY_DB_CONNECTION_TARGET ":memory:" - static const char* s_CreateSimpleTestTableSQL = R"( CREATE TABLE [main].[simpletest]( [first] INT, diff --git a/src/AppInstallerCLITests/TestCommon.cpp b/src/AppInstallerCLITests/TestCommon.cpp @@ -32,6 +32,8 @@ namespace TestCommon } static bool s_TempFileDestructorKeepsFile{}; + + static std::filesystem::path s_TestDataFileBasePath{}; } TempFile::TempFile(const std::string& baseName, const std::string& baseExt, bool deleteFileOnConstruction) @@ -55,4 +57,16 @@ namespace TestCommon { s_TempFileDestructorKeepsFile = keepFilesOnDestruction; } + + std::filesystem::path TestDataFile::GetPath() const + { + std::filesystem::path result = s_TestDataFileBasePath; + result /= m_path; + return result; + } + + void TestDataFile::SetTestDataBasePath(const std::filesystem::path& path) + { + s_TestDataFileBasePath = path; + } } diff --git a/src/AppInstallerCLITests/TestCommon.h b/src/AppInstallerCLITests/TestCommon.h @@ -2,10 +2,16 @@ // Licensed under the MIT License. #pragma once #include "pch.h" +#include <filesystem> #include <string> +#define SQLITE_MEMORY_DB_CONNECTION_TARGET ":memory:" + +#define REQUIRE_THROWS_HR(_expr_, _hr_) REQUIRE_THROWS_MATCHES(_expr_, wil::ResultException, ::TestCommon::ResultExceptionHRMatcher(_hr_)) + namespace TestCommon { + // Use this to create a temporary file for testing. struct TempFile { TempFile(const std::string& baseName, const std::string& baseExt, bool deleteFileOnConstruction = true); @@ -26,4 +32,39 @@ namespace TestCommon private: std::string _filepath; }; + + // Use this to find a test data file when testing. + struct TestDataFile + { + TestDataFile(const std::filesystem::path& path) : m_path(path) {} + + std::filesystem::path GetPath() const; + operator std::filesystem::path () const { return GetPath(); } + + static void SetTestDataBasePath(const std::filesystem::path& path); + + private: + std::filesystem::path m_path; + }; + + // Matcher that lets us verify wil::ResultExceptions have a specific HR. + struct ResultExceptionHRMatcher : public Catch::MatcherBase<wil::ResultException> + { + ResultExceptionHRMatcher(HRESULT hr) : m_expectedHR(hr) {} + + bool match(const wil::ResultException& re) const override + { + return re.GetErrorCode() == m_expectedHR; + } + + std::string describe() const override + { + std::ostringstream result; + result << "has HR == 0x" << std::hex << std::setfill('0') << std::setw(8) << m_expectedHR; + return result.str(); + } + + private: + HRESULT m_expectedHR = S_OK; + }; } diff --git a/src/AppInstallerCLITests/TestData/GoodManifest.yml b/src/AppInstallerCLITests/TestData/GoodManifest.yml @@ -1,8 +1,8 @@ Id: microsoft.msixsdk Name: MSIX SDK -ShortId: msixsdk +AppMoniker: msixsdk Version: 1.7.32 -CompanyName: Microsoft +Publisher: Microsoft Channel: release Author: Microsoft License: MIT License diff --git a/src/AppInstallerCLITests/YamlManifest.cpp b/src/AppInstallerCLITests/YamlManifest.cpp @@ -1,21 +1,42 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. #include "pch.h" +#include "TestCommon.h" #include "Manifest/Manifest.h" #include "AppInstallerSHA256.h" +using namespace TestCommon; using namespace AppInstaller::Manifest; using namespace AppInstaller::Utility; +using MultiValue = std::vector<std::string>; +bool operator==(const MultiValue& a, const MultiValue& b) +{ + if (a.size() != b.size()) + { + return false; + } + + for (size_t i = 0; i < a.size(); ++i) + { + if (a[i] != b[i]) + { + return false; + } + } + + return true; +} + TEST_CASE("ReadGoodManifestAndVerifyContents", "[PackageManifestHelper]") { - Manifest manifest = Manifest::CreateFromPath("GoodManifest.yml"); + Manifest manifest = Manifest::CreateFromPath(TestDataFile("GoodManifest.yml")); REQUIRE(manifest.Id == "microsoft.msixsdk"); REQUIRE(manifest.Name == "MSIX SDK"); - REQUIRE(manifest.ShortId == "msixsdk"); + REQUIRE(manifest.AppMoniker == "msixsdk"); REQUIRE(manifest.Version == "1.7.32"); - REQUIRE(manifest.CompanyName == "Microsoft"); + REQUIRE(manifest.Publisher == "Microsoft"); REQUIRE(manifest.Channel == "release"); REQUIRE(manifest.Author == "Microsoft"); REQUIRE(manifest.License == "MIT License"); @@ -23,10 +44,10 @@ TEST_CASE("ReadGoodManifestAndVerifyContents", "[PackageManifestHelper]") REQUIRE(manifest.MinOSVersion == "0.0.0.0"); REQUIRE(manifest.Description == "The MSIX SDK project is an effort to enable developers"); REQUIRE(manifest.Homepage == "https://github.com/microsoft/msix-packaging"); - REQUIRE(manifest.Tags == "msix,appx"); - REQUIRE(manifest.Commands == "makemsix,makeappx"); - REQUIRE(manifest.Protocols == "protocol1,protocol2"); - REQUIRE(manifest.FileExtensions == "appx,appxbundle,msix,msixbundle"); + REQUIRE(manifest.Tags == MultiValue{ "msix", "appx" }); + REQUIRE(manifest.Commands == MultiValue{ "makemsix", "makeappx" }); + REQUIRE(manifest.Protocols == MultiValue{ "protocol1", "protocol2" }); + REQUIRE(manifest.FileExtensions == MultiValue{ "appx", "appxbundle", "msix", "msixbundle" }); REQUIRE(manifest.InstallerType == "Zip"); // default switches @@ -78,5 +99,5 @@ TEST_CASE("ReadGoodManifestAndVerifyContents", "[PackageManifestHelper]") TEST_CASE("ReadBadManifestAndVerifyThrow", "[PackageManifestHelper]") { - REQUIRE_THROWS_WITH(Manifest::CreateFromPath("BadManifest-MissingName.yml"), Catch::Contains("invalid node; first invalid key: \"Name\"")); + REQUIRE_THROWS_WITH(Manifest::CreateFromPath(TestDataFile("BadManifest-MissingName.yml")), Catch::Contains("invalid node; first invalid key: \"Name\"")); } diff --git a/src/AppInstallerCLITests/main.cpp b/src/AppInstallerCLITests/main.cpp @@ -15,10 +15,24 @@ using namespace winrt; using namespace Windows::Foundation; using namespace std::string_literals; +// Logs the the AppInstaller log target to break up individual tests +struct LoggingBreakListener : public Catch::TestEventListenerBase +{ + using TestEventListenerBase::TestEventListenerBase; + + void testCaseStarting(const Catch::TestCaseInfo& info) override + { + AICLI_LOG(Test, Info, << "========== Test Case Begins :: " << info.name << " =========="); + } +}; +CATCH_REGISTER_LISTENER(LoggingBreakListener); + int main(int argc, char** argv) { init_apartment(); + bool hasSetTestDataBasePath = false; + std::vector<char*> args; for (int i = 0; i < argc; ++i) { @@ -28,7 +42,16 @@ int main(int argc, char** argv) } else if ("-log"s == argv[i]) { - AppInstaller::Logging::AddDefaultFileLogger(); + AppInstaller::Logging::AddFileLogger(); + } + else if ("-tdd"s == argv[i]) + { + ++i; + if (i < argc) + { + TestCommon::TestDataFile::SetTestDataBasePath(argv[i]); + hasSetTestDataBasePath = true; + } } else { @@ -36,6 +59,19 @@ int main(int argc, char** argv) } } + // If not set, use the current executables path + if (!hasSetTestDataBasePath) + { + wchar_t fullFileName[1024]; + DWORD chars = ARRAYSIZE(fullFileName); + if (QueryFullProcessImageNameW(GetCurrentProcess(), 0, fullFileName, &chars)) + { + std::filesystem::path filepath{ fullFileName }; + filepath.remove_filename(); + TestCommon::TestDataFile::SetTestDataBasePath(filepath); + } + } + // Enable all logging, to force log string building to run. // By not creating a log target, it will all be thrown away. AppInstaller::Logging::Log().EnableChannel(AppInstaller::Logging::Channel::All); diff --git a/src/AppInstallerCLITests/pch.h b/src/AppInstallerCLITests/pch.h @@ -12,11 +12,12 @@ #include <wil/result_macros.h> +#include <filesystem> +#include <fstream> +#include <future> #include <iostream> #include <sstream> #include <utility> -#include <filesystem> -#include <future> -#include <fstream> +#include <vector> #include <yaml-cpp/yaml.h> \ No newline at end of file diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj @@ -169,6 +169,7 @@ <ClInclude Include="pch.h" /> <ClInclude Include="Public\AppInstallerDownloader.h" /> <ClInclude Include="Public\AppInstallerErrors.h" /> + <ClInclude Include="Public\AppInstallerLanguageUtilities.h" /> <ClInclude Include="Public\AppInstallerRuntime.h" /> <ClInclude Include="Public\AppInstallerSHA256.h" /> <ClInclude Include="Public\AppInstallerStrings.h" /> diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters @@ -63,6 +63,9 @@ <ClInclude Include="Public\AppInstallerErrors.h"> <Filter>Public</Filter> </ClInclude> + <ClInclude Include="Public\AppInstallerLanguageUtilities.h"> + <Filter>Public</Filter> + </ClInclude> </ItemGroup> <ItemGroup> <ClCompile Include="pch.cpp"> diff --git a/src/AppInstallerCommonCore/AppInstallerLogging.cpp b/src/AppInstallerCommonCore/AppInstallerLogging.cpp @@ -4,6 +4,7 @@ #include "Public/AppInstallerLogging.h" #include "Public/AppInstallerTelemetry.h" +#include "DateTime.h" #include "FileLogger.h" namespace AppInstaller::Logging @@ -38,6 +39,7 @@ namespace AppInstaller::Logging case Channel::SQL: return "SQL"; case Channel::Repo: return "REPO"; case Channel::YAML: return "YAML"; + case Channel::Test: return "TEST"; default: return "NONE"; } } @@ -106,8 +108,14 @@ namespace AppInstaller::Logging } } - void AddDefaultFileLogger() + void AddFileLogger(const std::filesystem::path& filePath) { - Log().AddLogger(std::make_unique<FileLogger>()); + Log().AddLogger(std::make_unique<FileLogger>(filePath)); } } + +std::ostream& operator<<(std::ostream& out, const std::chrono::system_clock::time_point& time) +{ + AppInstaller::Utility::OutputTimepoint(out, time); + return out; +} diff --git a/src/AppInstallerCommonCore/FileLogger.cpp b/src/AppInstallerCommonCore/FileLogger.cpp @@ -46,8 +46,7 @@ namespace AppInstaller::Logging void FileLogger::Write(Channel channel, Level, std::string_view message) noexcept try { - Utility::OutputTimepoint(m_stream, std::chrono::system_clock::now()); - m_stream << " [" << std::setw(GetMaxChannelNameLength()) << std::left << std::setfill(' ') << GetChannelName(channel) << "] " << message << std::endl; + m_stream << std::chrono::system_clock::now() << " [" << std::setw(GetMaxChannelNameLength()) << std::left << std::setfill(' ') << GetChannelName(channel) << "] " << message << std::endl; } catch (...) { diff --git a/src/AppInstallerCommonCore/Public/AppInstallerErrors.h b/src/AppInstallerCommonCore/Public/AppInstallerErrors.h @@ -5,11 +5,13 @@ #define APPINSTALLER_CLI_ERROR_FACILITY 0x8A150000 -#define APPINSTALLER_CLI_ERROR_INTERNAL_ERROR 0x8A150001 -#define APPINSTALLER_CLI_ERROR_INVALID_CL_ARGUMENTS 0x8A150002 -#define APPINSTALLER_CLI_ERROR_COMMAND_FAILED 0x8A150003 -#define APPINSTALLER_CLI_ERROR_MANIFEST_FAILED 0x8A150004 -#define APPINSTALLER_CLI_ERROR_WORKFLOW_FAILED 0x8A150005 -#define APPINSTALLER_CLI_ERROR_INSTALLFLOW_FAILED 0x8A150006 -#define APPINSTALLER_CLI_ERROR_RUNTIME_ERROR 0x8A150007 -#define APPINSTALLER_CLI_ERROR_DOWNLOAD_FAILED 0x8A150008- \ No newline at end of file +#define APPINSTALLER_CLI_ERROR_INTERNAL_ERROR ((HRESULT)0x8A150001) +#define APPINSTALLER_CLI_ERROR_INVALID_CL_ARGUMENTS ((HRESULT)0x8A150002) +#define APPINSTALLER_CLI_ERROR_COMMAND_FAILED ((HRESULT)0x8A150003) +#define APPINSTALLER_CLI_ERROR_MANIFEST_FAILED ((HRESULT)0x8A150004) +#define APPINSTALLER_CLI_ERROR_WORKFLOW_FAILED ((HRESULT)0x8A150005) +#define APPINSTALLER_CLI_ERROR_INSTALLFLOW_FAILED ((HRESULT)0x8A150006) +#define APPINSTALLER_CLI_ERROR_RUNTIME_ERROR ((HRESULT)0x8A150007) +#define APPINSTALLER_CLI_ERROR_DOWNLOAD_FAILED ((HRESULT)0x8A150008) +#define APPINSTALLER_CLI_ERROR_CANNOT_WRITE_TO_UPLEVEL_INDEX ((HRESULT)0x8A150009) +#define APPINSTALLER_CLI_ERROR_INDEX_INTEGRITY_COMPROMISED ((HRESULT)0x8A15000A) diff --git a/src/AppInstallerCommonCore/Public/AppInstallerLanguageUtilities.h b/src/AppInstallerCommonCore/Public/AppInstallerLanguageUtilities.h @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once + +#include <initializer_list> +#include <string> +#include <string_view> +#include <tuple> +#include <type_traits> +#include <utility> + +namespace AppInstaller +{ + // A helper type that resets itself when it is moved from. + template <typename T> + struct ResetWhenMovedFrom + { + ResetWhenMovedFrom() = default; + + ResetWhenMovedFrom(T t) : m_var(t) {} + + // Not copyable + ResetWhenMovedFrom(const ResetWhenMovedFrom&) = delete; + ResetWhenMovedFrom& operator=(const ResetWhenMovedFrom&) = delete; + + ResetWhenMovedFrom(ResetWhenMovedFrom&& other) : + m_var(std::move(other.m_var)) + { + other.m_var = T{}; + } + + ResetWhenMovedFrom& operator=(ResetWhenMovedFrom&& other) + { + m_var = std::move(other.m_var); + other.m_var = T{}; + return *this; + } + + operator T& () { return m_var; } + operator const T& () const { return m_var; } + + private: + T m_var; + }; + + // Enables a bool to be used as a destruction indicator. + using DestructionToken = ResetWhenMovedFrom<bool>; +} diff --git a/src/AppInstallerCommonCore/Public/AppInstallerLogging.h b/src/AppInstallerCommonCore/Public/AppInstallerLogging.h @@ -2,6 +2,8 @@ // Licensed under the MIT License. #pragma once +#include <chrono> +#include <filesystem> #include <memory> #include <sstream> #include <string> @@ -32,6 +34,7 @@ namespace AppInstaller::Logging SQL, Repo, YAML, + Test, All, }; @@ -116,5 +119,8 @@ namespace AppInstaller::Logging } // Adds the default file logger to the DiagnosticLogger. - void AddDefaultFileLogger(); + void AddFileLogger(const std::filesystem::path& filePath = {}); } + +// Enable output of system_clock timepoints. +std::ostream& operator<<(std::ostream& out, const std::chrono::system_clock::time_point& time); diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -187,6 +187,8 @@ <ClInclude Include="Microsoft\Schema\Version.h" /> <ClInclude Include="Microsoft\SQLiteIndex.h" /> <ClInclude Include="pch.h" /> + <ClInclude Include="Public\AppInstallerRepositorySearch.h" /> + <ClInclude Include="Public\AppInstallerRepositorySource.h" /> <ClInclude Include="SQLiteWrapper.h" /> </ItemGroup> <ItemGroup> diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -25,6 +25,9 @@ <Filter Include="Manifest"> <UniqueIdentifier>{052eaac1-ec6e-4879-8388-aff5a30401f0}</UniqueIdentifier> </Filter> + <Filter Include="Public"> + <UniqueIdentifier>{69ce2e35-fe7f-41af-bd47-91a70131d167}</UniqueIdentifier> + </Filter> </ItemGroup> <ItemGroup> <ClInclude Include="pch.h"> @@ -99,6 +102,12 @@ <ClInclude Include="Microsoft\Schema\1_0\PathPartTable.h"> <Filter>Microsoft\Schema\1_0</Filter> </ClInclude> + <ClInclude Include="Public\AppInstallerRepositorySource.h"> + <Filter>Public</Filter> + </ClInclude> + <ClInclude Include="Public\AppInstallerRepositorySearch.h"> + <Filter>Public</Filter> + </ClInclude> </ItemGroup> <ItemGroup> <ClCompile Include="pch.cpp"> diff --git a/src/AppInstallerRepositoryCore/Manifest/Manifest.cpp b/src/AppInstallerRepositoryCore/Manifest/Manifest.cpp @@ -5,25 +5,56 @@ namespace AppInstaller::Manifest { + namespace + { + std::vector<std::string> SplitMultiValueField(const std::string& input) + { + if (input.empty()) + { + return {}; + } + + std::vector<std::string> result; + size_t currentPos = 0; + while (currentPos < input.size()) + { + size_t splitPos = input.find(',', currentPos); + if (splitPos == std::string::npos) + { + splitPos = input.size(); + } + + std::string splitVal = input.substr(currentPos, splitPos - currentPos); + if (!splitVal.empty()) + { + result.emplace_back(std::move(splitVal)); + } + currentPos = splitPos + 1; + } + + return result; + } + } + void Manifest::PopulateManifestFields(const YAML::Node& rootNode) { // Required fields this->Id = rootNode["Id"].as<std::string>(); this->Name = rootNode["Name"].as<std::string>(); this->Version = rootNode["Version"].as<std::string>(); + this->Publisher = rootNode["Publisher"].as<std::string>(); // Optional fields. - this->ShortId = rootNode["ShortId"] ? rootNode["ShortId"].as<std::string>() : ""; - this->CompanyName = rootNode["CompanyName"] ? rootNode["CompanyName"].as<std::string>() : ""; + this->AppMoniker = rootNode["AppMoniker"] ? rootNode["AppMoniker"].as<std::string>() : ""; this->Authors = rootNode["Authors"] ? rootNode["Authors"].as<std::string>() : ""; this->Channel = rootNode["Channel"] ? rootNode["Channel"].as<std::string>() : ""; this->Author = rootNode["Author"] ? rootNode["Author"].as<std::string>() : ""; this->License = rootNode["License"] ? rootNode["License"].as<std::string>() : ""; this->MinOSVersion = rootNode["MinOSVersion"] ? rootNode["MinOSVersion"].as<std::string>() : ""; - this->Tags = rootNode["Tags"] ? rootNode["Tags"].as<std::string>() : ""; - this->Commands = rootNode["Commands"] ? rootNode["Commands"].as<std::string>() : ""; - this->Protocols = rootNode["Protocols"] ? rootNode["Protocols"].as<std::string>() : ""; - this->FileExtensions = rootNode["FileExtensions"] ? rootNode["FileExtensions"].as<std::string>() : ""; + this->Tags = SplitMultiValueField(rootNode["Tags"] ? rootNode["Tags"].as<std::string>() : ""); + this->Commands = SplitMultiValueField(rootNode["Commands"] ? rootNode["Commands"].as<std::string>() : ""); + this->Protocols = SplitMultiValueField(rootNode["Protocols"] ? rootNode["Protocols"].as<std::string>() : ""); + this->FileExtensions = SplitMultiValueField(rootNode["FileExtensions"] ? rootNode["FileExtensions"].as<std::string>() : ""); this->InstallerType = rootNode["InstallerType"] ? rootNode["InstallerType"].as<std::string>() : ""; this->Description = rootNode["Description"] ? rootNode["Description"].as<std::string>() : ""; this->Homepage = rootNode["Homepage"] ? rootNode["Homepage"].as<std::string>() : ""; diff --git a/src/AppInstallerRepositoryCore/Manifest/Manifest.h b/src/AppInstallerRepositoryCore/Manifest/Manifest.h @@ -30,10 +30,9 @@ namespace AppInstaller::Manifest // Required std::string Version; - // Name subject to change - std::string ShortId; + std::string AppMoniker; - std::string CompanyName; + std::string Publisher; // Comma separated Values std::string Authors; @@ -47,16 +46,16 @@ namespace AppInstaller::Manifest std::string MinOSVersion; // Comma separated values - std::string Tags; + std::vector<std::string> Tags; // Comma separated values - std::string Commands; + std::vector<std::string> Commands; // Comma separated values - std::string Protocols; + std::vector<std::string> Protocols; // Comma separated values - std::string FileExtensions; + std::vector<std::string> FileExtensions; std::vector<ManifestInstaller> Installers; diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp @@ -38,6 +38,8 @@ namespace AppInstaller::Repository::Microsoft result.m_interface->CreateTables(result.m_dbconn); + result.SetLastWriteTime(); + savepoint.Commit(); return result; @@ -108,8 +110,9 @@ namespace AppInstaller::Repository::Microsoft m_dbconn(SQLite::Connection::Create(target, disposition, flags)) { m_version = Schema::Version::GetSchemaVersion(m_dbconn); - AICLI_LOG(Repo, Info, << "Opened SQLite Index with version: " << m_version); + AICLI_LOG(Repo, Info, << "Opened SQLite Index with version [" << m_version << "], last write [" << GetLastWriteTime() << "]"); m_interface = m_version.CreateISQLiteIndex(); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_CANNOT_WRITE_TO_UPLEVEL_INDEX, disposition == SQLite::Connection::OpenDisposition::ReadWrite && m_version != m_interface->GetVersion()); } SQLiteIndex::SQLiteIndex(const std::string& target, Schema::Version version) : @@ -121,43 +124,86 @@ namespace AppInstaller::Repository::Microsoft void SQLiteIndex::AddManifest(const std::filesystem::path& manifestPath, const std::filesystem::path& relativePath) { + AICLI_LOG(Repo, Info, << "Adding manifest from file [" << manifestPath << "]"); + Manifest::Manifest manifest = Manifest::Manifest::CreateFromPath(manifestPath); AddManifest(manifest, relativePath); } void SQLiteIndex::AddManifest(const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) { - UNREFERENCED_PARAMETER(manifest); - UNREFERENCED_PARAMETER(relativePath); - THROW_HR(E_NOTIMPL); + AICLI_LOG(Repo, Info, << "Adding manifest for [" << manifest.Id << ", " << manifest.Version << "] at relative path [" << relativePath << "]"); + + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "sqliteindex_addmanifest"); + + m_interface->AddManifest(m_dbconn, manifest, relativePath); + + SetLastWriteTime(); + + savepoint.Commit(); } - void SQLiteIndex::UpdateManifest(const std::filesystem::path& oldManifestPath, const std::filesystem::path& oldRelativePath, const std::filesystem::path& newManifestPath, const std::filesystem::path& newRelativePath) + bool SQLiteIndex::UpdateManifest(const std::filesystem::path& oldManifestPath, const std::filesystem::path& oldRelativePath, const std::filesystem::path& newManifestPath, const std::filesystem::path& newRelativePath) { + AICLI_LOG(Repo, Info, << "Updating manifest from file [" << oldManifestPath << "] to file [" << newManifestPath << "]"); + Manifest::Manifest oldManifest = Manifest::Manifest::CreateFromPath(oldManifestPath); Manifest::Manifest newManifest = Manifest::Manifest::CreateFromPath(newManifestPath); - UpdateManifest(oldManifest, oldRelativePath, newManifest, newRelativePath); + return UpdateManifest(oldManifest, oldRelativePath, newManifest, newRelativePath); } - void SQLiteIndex::UpdateManifest(const Manifest::Manifest& oldManifest, const std::filesystem::path& oldRelativePath, const Manifest::Manifest& newManifest, const std::filesystem::path& newRelativePath) + bool SQLiteIndex::UpdateManifest(const Manifest::Manifest& oldManifest, const std::filesystem::path& oldRelativePath, const Manifest::Manifest& newManifest, const std::filesystem::path& newRelativePath) { - UNREFERENCED_PARAMETER(oldManifest); - UNREFERENCED_PARAMETER(oldRelativePath); - UNREFERENCED_PARAMETER(newManifest); - UNREFERENCED_PARAMETER(newRelativePath); - THROW_HR(E_NOTIMPL); + AICLI_LOG(Repo, Info, << "Updating manifest from [" << oldManifest.Id << ", " << oldManifest.Version << "] to [" << newManifest.Id << ", " << newManifest.Version << + "] at relative path [" << oldRelativePath << "] to [" << newRelativePath << "]"); + + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "sqliteindex_updatemanifest"); + + bool result = m_interface->UpdateManifest(m_dbconn, oldManifest, oldRelativePath, newManifest, newRelativePath); + + if (result) + { + SetLastWriteTime(); + + savepoint.Commit(); + } + + return result; } void SQLiteIndex::RemoveManifest(const std::filesystem::path& manifestPath, const std::filesystem::path& relativePath) { + AICLI_LOG(Repo, Info, << "Removing manifest from file [" << manifestPath << "]"); + Manifest::Manifest manifest = Manifest::Manifest::CreateFromPath(manifestPath); RemoveManifest(manifest, relativePath); } void SQLiteIndex::RemoveManifest(const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) { - UNREFERENCED_PARAMETER(manifest); - UNREFERENCED_PARAMETER(relativePath); - THROW_HR(E_NOTIMPL); + AICLI_LOG(Repo, Info, << "Removing manifest for [" << manifest.Id << ", " << manifest.Version << "] at relative path [" << relativePath << "]"); + + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(m_dbconn, "sqliteindex_removemanifest"); + + m_interface->RemoveManifest(m_dbconn, manifest, relativePath); + + SetLastWriteTime(); + + savepoint.Commit(); + } + + // Recording last write time based on MSDN documentation stating that time returns a POSIX epoch time and thus + // should be consistent across systems. + void SQLiteIndex::SetLastWriteTime() + { + static_assert(std::is_same_v<int64_t, decltype(time(nullptr))>, "time returns a 64-bit integer"); + time_t now = time(nullptr); + Schema::MetadataTable::SetNamedValue(m_dbconn, Schema::s_MetadataValueName_LastWriteTime, static_cast<int64_t>(now)); + } + + std::chrono::system_clock::time_point SQLiteIndex::GetLastWriteTime() + { + int64_t lastWriteTime = Schema::MetadataTable::GetNamedValue<int64_t>(m_dbconn, Schema::s_MetadataValueName_LastWriteTime); + return std::chrono::system_clock::from_time_t(static_cast<time_t>(lastWriteTime)); } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h @@ -5,6 +5,7 @@ #include "Microsoft/Schema/Version.h" #include "Manifest/Manifest.h" +#include <chrono> #include <filesystem> #include <limits> #include <memory> @@ -41,19 +42,24 @@ namespace AppInstaller::Repository::Microsoft // Gets the schema version of the index. Schema::Version GetVersion() const { return m_version; } + // Gets the last write time for the index. + std::chrono::system_clock::time_point GetLastWriteTime(); + // Adds the manifest at the repository relative path to the index. + // If the function succeeds, the manifest has been added. void AddManifest(const std::filesystem::path& manifestPath, const std::filesystem::path& relativePath); // Adds the manifest at the repository relative path to the index. + // If the function succeeds, the manifest has been added. void AddManifest(const Manifest::Manifest& manifest, const std::filesystem::path& relativePath); // Updates the manifest at the repository relative path in the index. - // If the old manifest does not exist in the index, this is equivalent to AddManifest(newManifest, newRelativePath). - void UpdateManifest(const std::filesystem::path& oldManifestPath, const std::filesystem::path& oldRelativePath, const std::filesystem::path& newManifestPath, const std::filesystem::path& newRelativePath); + // The return value indicates whether the index was modified by the function. + bool UpdateManifest(const std::filesystem::path& oldManifestPath, const std::filesystem::path& oldRelativePath, const std::filesystem::path& newManifestPath, const std::filesystem::path& newRelativePath); // Updates the manifest at the repository relative path in the index. - // If the old manifest does not exist in the index, this is equivalent to AddManifest(newManifest, newRelativePath). - void UpdateManifest(const Manifest::Manifest& oldManifest, const std::filesystem::path& oldRelativePath, const Manifest::Manifest& newManifest, const std::filesystem::path& newRelativePath); + // The return value indicates whether the index was modified by the function. + bool UpdateManifest(const Manifest::Manifest& oldManifest, const std::filesystem::path& oldRelativePath, const Manifest::Manifest& newManifest, const std::filesystem::path& newRelativePath); // Removes the manifest at the repository relative path from the index. void RemoveManifest(const std::filesystem::path& manifestPath, const std::filesystem::path& relativePath); @@ -68,6 +74,9 @@ namespace AppInstaller::Repository::Microsoft // Constructor used to create a new index. SQLiteIndex(const std::string& target, Schema::Version version); + // Sets the last write time metadata value in the index. + void SetLastWriteTime(); + SQLite::Connection m_dbconn; Schema::Version m_version; std::unique_ptr<Schema::ISQLiteIndex> m_interface; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ChannelTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ChannelTable.h @@ -12,8 +12,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 struct ChannelTableInfo { - inline static std::string_view TableName() { return "channels"sv; } - inline static std::string_view ValueName() { return "channel"sv; } + inline static constexpr std::string_view TableName() { return "channels"sv; } + inline static constexpr std::string_view ValueName() { return "channel"sv; } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/CommandsTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/CommandsTable.h @@ -12,8 +12,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 struct CommandsTableInfo { - inline static std::string_view TableName() { return "commands"sv; } - inline static std::string_view ValueName() { return "command"sv; } + inline static constexpr std::string_view TableName() { return "commands"sv; } + inline static constexpr std::string_view ValueName() { return "command"sv; } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ExtensionsTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ExtensionsTable.h @@ -12,8 +12,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 struct ExtensionsTableInfo { - inline static std::string_view TableName() { return "extensions"sv; } - inline static std::string_view ValueName() { return "extension"sv; } + inline static constexpr std::string_view TableName() { return "extensions"sv; } + inline static constexpr std::string_view ValueName() { return "extension"sv; } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/IdTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/IdTable.h @@ -12,8 +12,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 struct IdTableInfo { - inline static std::string_view TableName() { return "ids"sv; } - inline static std::string_view ValueName() { return "id"sv; } + inline static constexpr std::string_view TableName() { return "ids"sv; } + inline static constexpr std::string_view ValueName() { return "id"sv; } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.cpp @@ -20,6 +20,65 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 { + namespace + { + // Holds the information on a potentially existing manifest. + struct ExistingManifestInfo + { + SQLite::rowid_t PathLeaf; + std::optional<SQLite::rowid_t> Manifest; + }; + + // Gets an existing manifest by its rowid. + // The return value contains the path leaf and manifest rowid, if they exist. + ExistingManifestInfo GetExistingManifestId(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) + { + auto [pathFound, pathLeafId] = PathPartTable::EnsurePathExists(connection, relativePath, false); + + // If we do not find the path, there can be no manifest + if (!pathFound) + { + AICLI_LOG(Repo, Info, << "Did not find a manifest to remove at path: " << relativePath.u8string()); + return {}; + } + + ExistingManifestInfo result{}; + result.PathLeaf = pathLeafId; + result.Manifest = ManifestTable::SelectByValueId<PathPartTable>(connection, result.PathLeaf); + + // If the manifest didn't actually exist, then remove the path + if (!result.Manifest) + { + AICLI_LOG(Repo, Info, << "Did not find a manifest row for the path: " << relativePath.u8string()); + PathPartTable::RemovePathById(connection, result.PathLeaf); + return {}; + } + + // Ensure that the given manifest matches the data in the index + auto [idValue, versionValue, channelValue] = ManifestTable::GetValuesById<IdTable, VersionTable, ChannelTable>(connection, result.Manifest.value()); + + if (idValue != manifest.Id) + { + AICLI_LOG(Repo, Error, << "Existing manifest in index does not match given value for Id: [" << idValue << "] != [" << manifest.Id << "]"); + THROW_HR(E_NOT_VALID_STATE); + } + + if (versionValue != manifest.Version) + { + AICLI_LOG(Repo, Error, << "Existing manifest in index does not match given value for Version: [" << versionValue << "] != [" << manifest.Version << "]"); + THROW_HR(E_NOT_VALID_STATE); + } + + if (channelValue != manifest.Channel) + { + AICLI_LOG(Repo, Error, << "Existing manifest in index does not match given value for Channel: [" << channelValue << "] != [" << manifest.Channel << "]"); + THROW_HR(E_NOT_VALID_STATE); + } + + return result; + } + } + Schema::Version Interface::GetVersion() const { return { 1, 0 }; @@ -27,7 +86,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 void Interface::CreateTables(SQLite::Connection& connection) { - SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "V1_0_createtables"); + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "createtables_v1_0"); IdTable::Create(connection); NameTable::Create(connection); @@ -37,8 +96,14 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 PathPartTable::Create(connection); - ManifestTable::Create(connection, - { IdTable::ValueName(), NameTable::ValueName(), MonikerTable::ValueName(), VersionTable::ValueName(), ChannelTable::ValueName(), PathPartTable::ValueName() }); + ManifestTable::Create(connection, { + { IdTable::ValueName(), true, false }, + { NameTable::ValueName(), false, false }, + { MonikerTable::ValueName(), false, false }, + { VersionTable::ValueName(), true, false }, + { ChannelTable::ValueName(), true, false }, + { PathPartTable::ValueName(), false, true } + }); TagsTable::Create(connection); CommandsTable::Create(connection); @@ -47,4 +112,88 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 savepoint.Commit(); } + + void Interface::AddManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "addmanifest_v1_0"); + + auto [pathFound, pathLeafId] = PathPartTable::EnsurePathExists(connection, relativePath, true); + + // If we get false from the function, this manifest already exists in the index. + THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_ALREADY_EXISTS), !pathFound); + + // Ensure that all of the 1:1 data exists. + SQLite::rowid_t idId = IdTable::EnsureExists(connection, manifest.Id); + SQLite::rowid_t nameId = NameTable::EnsureExists(connection, manifest.Name); + SQLite::rowid_t monikerId = MonikerTable::EnsureExists(connection, manifest.AppMoniker); + SQLite::rowid_t versionId = VersionTable::EnsureExists(connection, manifest.Version); + SQLite::rowid_t channelId = ChannelTable::EnsureExists(connection, manifest.Channel); + + // Insert the manifest entry. + SQLite::rowid_t manifestId = ManifestTable::Insert(connection, { + { IdTable::ValueName(), idId}, + { NameTable::ValueName(), nameId }, + { MonikerTable::ValueName(), monikerId }, + { VersionTable::ValueName(), versionId }, + { ChannelTable::ValueName(), channelId }, + { PathPartTable::ValueName(), pathLeafId } + }); + + // Add all of the 1:N data. + TagsTable::EnsureExistsAndInsert(connection, manifest.Tags, manifestId); + CommandsTable::EnsureExistsAndInsert(connection, manifest.Commands, manifestId); + ProtocolsTable::EnsureExistsAndInsert(connection, manifest.Protocols, manifestId); + ExtensionsTable::EnsureExistsAndInsert(connection, manifest.FileExtensions, manifestId); + + savepoint.Commit(); + } + + bool Interface::UpdateManifest(SQLite::Connection& connection, + const Manifest::Manifest& oldManifest, const std::filesystem::path& oldRelativePath, + const Manifest::Manifest& newManifest, const std::filesystem::path& newRelativePath) + { + UNREFERENCED_PARAMETER(connection); + UNREFERENCED_PARAMETER(oldManifest); + UNREFERENCED_PARAMETER(oldRelativePath); + UNREFERENCED_PARAMETER(newManifest); + UNREFERENCED_PARAMETER(newRelativePath); + THROW_HR(E_NOTIMPL); + } + + void Interface::RemoveManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "removemanifest_v1_0"); + + ExistingManifestInfo manifestInfo = GetExistingManifestId(connection, manifest, relativePath); + + // If the manifest doesn't actually exist, fail the remove. + THROW_HR_IF(E_NOT_SET, !manifestInfo.Manifest); + + SQLite::rowid_t manifestId = manifestInfo.Manifest.value(); + + // Get the ids of the values from the manifest table + auto [idId, nameId, monikerId, versionId, channelId] = + ManifestTable::GetIdsById<IdTable, NameTable, MonikerTable, VersionTable, ChannelTable>(connection, manifestId); + + // Remove the manifest row + ManifestTable::DeleteById(connection, manifestId); + + // Remove all of the 1:1 data that is no longer referenced. + IdTable::DeleteIfNotNeededById(connection, idId); + NameTable::DeleteIfNotNeededById(connection, nameId); + MonikerTable::DeleteIfNotNeededById(connection, monikerId); + VersionTable::DeleteIfNotNeededById(connection, versionId); + ChannelTable::DeleteIfNotNeededById(connection, channelId); + + // Remove the path + PathPartTable::RemovePathById(connection, manifestInfo.PathLeaf); + + // Remove all of the 1:N data that is no longer referenced. + TagsTable::DeleteIfNotNeededByManifestId(connection, manifestId); + CommandsTable::DeleteIfNotNeededByManifestId(connection, manifestId); + ProtocolsTable::DeleteIfNotNeededByManifestId(connection, manifestId); + ExtensionsTable::DeleteIfNotNeededByManifestId(connection, manifestId); + + savepoint.Commit(); + } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h @@ -12,5 +12,10 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Version 1.0 Schema::Version GetVersion() const override; void CreateTables(SQLite::Connection& connection) override; + void AddManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) override; + bool UpdateManifest(SQLite::Connection& connection, + const Manifest::Manifest& oldManifest, const std::filesystem::path& oldRelativePath, + const Manifest::Manifest& newManifest, const std::filesystem::path& newRelativePath) override; + void RemoveManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) override; }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.cpp @@ -9,22 +9,197 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 using namespace std::string_view_literals; static constexpr std::string_view s_ManifestTable_Table_Name = "manifest"sv; - void ManifestTable::Create(SQLite::Connection& connection, std::initializer_list<std::string_view> values) + namespace details { + std::optional<SQLite::rowid_t> ManifestTableSelectByValueId(SQLite::Connection& connection, std::string_view valueName, SQLite::rowid_t id) + { + std::ostringstream selectSQL; + selectSQL << "SELECT [" << SQLite::RowIDName << "] FROM [" << s_ManifestTable_Table_Name << "] WHERE [" << valueName << "] = ? LIMIT 1"; + + SQLite::Statement select = SQLite::Statement::Create(connection, selectSQL.str()); + + select.Bind(1, id); + + if (select.Step()) + { + return select.GetColumn<SQLite::rowid_t>(0); + } + else + { + return {}; + } + } + + SQLite::Statement ManifestTableGetIdsById_Statement( + SQLite::Connection& connection, + SQLite::rowid_t id, + std::initializer_list<std::string_view> values) + { + std::ostringstream selectSQL; + selectSQL << "SELECT "; + + // add columns to select + bool isFirst = true; + for (const std::string_view& value : values) + { + selectSQL << (isFirst ? "[" : ", [") << value << ']'; + isFirst = false; + } + + selectSQL << " FROM [" << s_ManifestTable_Table_Name << "] WHERE [" << SQLite::RowIDName << "] = ?"; + + SQLite::Statement result = SQLite::Statement::Create(connection, selectSQL.str()); + + result.Bind(1, id); + + THROW_HR_IF(E_NOT_SET, !result.Step()); + + return result; + } + + // Creates a statement and executes it, select the actual values for a given manifest id. + // Ex. + // SELECT [ids].[id] FROM [manifest] + // JOIN [ids] ON [manifest].[id] = [ids].[rowid] + // WHERE [manifest].[rowid] = 1 + SQLite::Statement ManifestTableGetValuesById_Statement( + SQLite::Connection& connection, + SQLite::rowid_t id, + std::initializer_list<ManifestOneToOneTableInfo> tableInfos) + { + std::ostringstream selectSQL; + selectSQL << "SELECT "; + + // add columns to select + bool isFirst = true; + for (const ManifestOneToOneTableInfo& tableInfo : tableInfos) + { + selectSQL << (isFirst ? "[" : ", [") << tableInfo.Table << "].[" << tableInfo.Value << ']'; + isFirst = false; + } + + selectSQL << " FROM [" << s_ManifestTable_Table_Name << "] "; + + // join tables + for (const ManifestOneToOneTableInfo& tableInfo : tableInfos) + { + selectSQL << "JOIN [" << tableInfo.Table << "] ON " << + '[' << s_ManifestTable_Table_Name << "].[" << tableInfo.Value << "] = [" << tableInfo.Table << "].[" << SQLite::RowIDName << "] "; + } + + selectSQL << " WHERE [" << s_ManifestTable_Table_Name << "].[" << SQLite::RowIDName << "] = ?"; + + SQLite::Statement result = SQLite::Statement::Create(connection, selectSQL.str()); + + result.Bind(1, id); + + THROW_HR_IF(E_NOT_SET, !result.Step()); + + return result; + } + } + + void ManifestTable::Create(SQLite::Connection& connection, std::initializer_list<ManifestColumnInfo> values) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "createManifestTable_v1_0"); + std::ostringstream createTableSQL; - createTableSQL << "CREATE TABLE [" << s_ManifestTable_Table_Name << "]("; + createTableSQL << "CREATE TABLE [" << s_ManifestTable_Table_Name << "] ("; + + for (const ManifestColumnInfo& value : values) + { + createTableSQL << '[' << value.Name << "] INT64 NOT NULL" << (value.Unique ? " UNIQUE" : "") << ","; + } + + createTableSQL << "PRIMARY KEY("; bool isFirst = true; - for (const std::string_view& value : values) + for (const ManifestColumnInfo& value : values) { - createTableSQL << (isFirst ? "[" : ",[") << value << "] INT64"; + if (value.PrimaryKey) + { + createTableSQL << (isFirst ? "[" : ", [") << value.Name << "]"; + } isFirst = false; } - createTableSQL << ')'; + createTableSQL << "))"; SQLite::Statement createStatement = SQLite::Statement::Create(connection, createTableSQL.str()); createStatement.Execute(); + + // Create an index on every value to improve performance + for (const ManifestColumnInfo& value : values) + { + std::ostringstream createIndexSQL; + createIndexSQL << "CREATE INDEX [" << s_ManifestTable_Table_Name << '_' << value.Name << "_index] " + << "ON [" << s_ManifestTable_Table_Name << "](" + << '[' << value.Name << "])"; + + SQLite::Statement createIndex = SQLite::Statement::Create(connection, createIndexSQL.str()); + + createIndex.Execute(); + } + + savepoint.Commit(); + } + + SQLite::rowid_t ManifestTable::Insert(SQLite::Connection& connection, std::initializer_list<ManifestOneToOneValue> values) + { + std::ostringstream insertSQL; + insertSQL << "INSERT INTO [" << s_ManifestTable_Table_Name << "] ("; + + bool isFirst = true; + for (const ManifestOneToOneValue& value : values) + { + insertSQL << (isFirst ? "[" : ",[") << value.Name << "] "; + isFirst = false; + } + + insertSQL << ") VALUES ("; + + for (size_t i = 0; i < values.size(); ++i) + { + insertSQL << (i == 0 ? "?" : ", ?"); + } + + insertSQL << ')'; + + SQLite::Statement insert = SQLite::Statement::Create(connection, insertSQL.str()); + + int bindIndex = 1; + for (const ManifestOneToOneValue& value : values) + { + insert.Bind(bindIndex++, value.Value); + } + + insert.Execute(); + + return connection.GetLastInsertRowID(); + } + + void ManifestTable::DeleteById(SQLite::Connection& connection, SQLite::rowid_t id) + { + std::ostringstream deleteSQL; + deleteSQL << "DELETE FROM [" << s_ManifestTable_Table_Name << "] WHERE [" << SQLite::RowIDName << "] = ?"; + + SQLite::Statement deleteStatement = SQLite::Statement::Create(connection, deleteSQL.str()); + + deleteStatement.Bind(1, id); + + deleteStatement.Execute(); + } + + bool ManifestTable::IsEmpty(SQLite::Connection& connection) + { + std::ostringstream countSQL; + countSQL << "SELECT COUNT(*) FROM [" << s_ManifestTable_Table_Name << ']'; + + SQLite::Statement countStatement = SQLite::Statement::Create(connection, countSQL.str()); + + THROW_HR_IF(E_UNEXPECTED, !countStatement.Step()); + + return (countStatement.GetColumn<int>(0) == 0); } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.h @@ -3,15 +3,86 @@ #pragma once #include "SQLiteWrapper.h" #include <initializer_list> +#include <optional> #include <string_view> namespace AppInstaller::Repository::Microsoft::Schema::V1_0 { + namespace details + { + // Table info that is 1:1 with the manifest. + struct ManifestOneToOneTableInfo + { + std::string_view Table; + std::string_view Value; + }; + + // Selects a manifest by the given value id. + std::optional<SQLite::rowid_t> ManifestTableSelectByValueId(SQLite::Connection& connection, std::string_view valueName, SQLite::rowid_t id); + + // Gets the requested ids for the manifest with the given rowid. + SQLite::Statement ManifestTableGetIdsById_Statement( + SQLite::Connection& connection, + SQLite::rowid_t id, + std::initializer_list<std::string_view> values); + + // Gets the requested values for the manifest with the given rowid. + SQLite::Statement ManifestTableGetValuesById_Statement( + SQLite::Connection& connection, + SQLite::rowid_t id, + std::initializer_list<ManifestOneToOneTableInfo> tableInfos); + } + + // Info on the manifest columns. + struct ManifestColumnInfo + { + std::string_view Name; + bool PrimaryKey; + bool Unique; + }; + + // A value that is 1:1 with the manifest. + struct ManifestOneToOneValue + { + std::string_view Name; + SQLite::rowid_t Value; + }; + // A table that represents a single manifest struct ManifestTable { // Creates the table. - static void Create(SQLite::Connection& connection, std::initializer_list<std::string_view> values); + static void Create(SQLite::Connection& connection, std::initializer_list<ManifestColumnInfo> values); + + // Insert the given values into the table. + static SQLite::rowid_t Insert(SQLite::Connection& connection, std::initializer_list<ManifestOneToOneValue> values); + + // Select the first rowid of the manifest with the given value. + template <typename Table> + static std::optional<SQLite::rowid_t> SelectByValueId(SQLite::Connection& connection, SQLite::rowid_t id) + { + return details::ManifestTableSelectByValueId(connection, Table::ValueName(), id); + } + + // Gets the ids requested for the manifest with the given rowid. + template <typename... Tables> + static auto GetIdsById(SQLite::Connection& connection, SQLite::rowid_t id) + { + return details::ManifestTableGetIdsById_Statement(connection, id, { Tables::ValueName()... }).GetRow<Tables::id_t...>(); + } + + // Gets the values requested for the manifest with the given rowid. + template <typename... Tables> + static auto GetValuesById(SQLite::Connection& connection, SQLite::rowid_t id) + { + return details::ManifestTableGetValuesById_Statement(connection, id, { details::ManifestOneToOneTableInfo{ Tables::TableName(), Tables::ValueName() }... }).GetRow<Tables::value_t...>(); + } + + // Deletes the manifest row with the given rowid. + static void DeleteById(SQLite::Connection& connection, SQLite::rowid_t id); + + // Determines if the table is empty. + static bool IsEmpty(SQLite::Connection& connection); }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/MonikerTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/MonikerTable.h @@ -12,8 +12,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 struct MonikerTableInfo { - inline static std::string_view TableName() { return "monikers"sv; } - inline static std::string_view ValueName() { return "moniker"sv; } + inline static constexpr std::string_view TableName() { return "monikers"sv; } + inline static constexpr std::string_view ValueName() { return "moniker"sv; } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/NameTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/NameTable.h @@ -12,8 +12,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 struct NameTableInfo { - inline static std::string_view TableName() { return "names"sv; } - inline static std::string_view ValueName() { return "name"sv; } + inline static constexpr std::string_view TableName() { return "names"sv; } + inline static constexpr std::string_view ValueName() { return "name"sv; } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.cpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include "Microsoft/Schema/1_0/OneToManyTable.h" +#include "Microsoft/Schema/1_0/OneToOneTable.h" namespace AppInstaller::Repository::Microsoft::Schema::V1_0 @@ -14,23 +15,17 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 void CreateOneToManyTable(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName) { - SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, std::string{ tableName } +"_create"); + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, std::string{ tableName } +"_create_v1_0"); - // Create the table itself - std::ostringstream createTableSQL; - createTableSQL << "CREATE TABLE [" << tableName << "](" - << '[' << valueName << "] TEXT PRIMARY KEY)"; - - SQLite::Statement createTableStatement = SQLite::Statement::Create(connection, createTableSQL.str()); - - createTableStatement.Execute(); + // Create the data table as a 1:1 + CreateOneToOneTable(connection, tableName, valueName); // Create the mapping table std::ostringstream createMapTableSQL; createMapTableSQL << "CREATE TABLE [" << tableName << s_OneToManyTable_MapTable_Suffix << "](" << "[" << s_OneToManyTable_MapTable_ManifestName << "] INT64 NOT NULL," << '[' << valueName << "] INT64 NOT NULL," - "UNIQUE([" << s_OneToManyTable_MapTable_ManifestName << "], [" << valueName << "]))"; + "PRIMARY KEY([" << s_OneToManyTable_MapTable_ManifestName << "], [" << valueName << "]))"; SQLite::Statement createMapStatement = SQLite::Statement::Create(connection, createMapTableSQL.str()); @@ -38,5 +33,112 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 savepoint.Commit(); } + + void OneToManyTableEnsureExistsAndInsert(SQLite::Connection& connection, + std::string_view tableName, std::string_view valueName, + const std::vector<std::string>& values, SQLite::rowid_t manifestId) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, std::string{ tableName } +"_ensureandinsert_v1_0"); + + // Create the mapping table insert statement for multiple use + std::ostringstream insertMappingSQL; + insertMappingSQL << "INSERT INTO [" << tableName << s_OneToManyTable_MapTable_Suffix << "] (" + << s_OneToManyTable_MapTable_ManifestName << ", " << valueName << ") VALUES (?, ?)"; + + SQLite::Statement insertMapping = SQLite::Statement::Create(connection, insertMappingSQL.str()); + insertMapping.Bind(1, manifestId); + + for (const std::string& value : values) + { + // First, ensure that the data exists + SQLite::rowid_t dataId = OneToOneTableEnsureExists(connection, tableName, valueName, value); + + // Second, insert into the mapping table + insertMapping.Reset(); + insertMapping.Bind(2, dataId); + + insertMapping.Execute(); + } + + savepoint.Commit(); + } + + void OneToManyTableDeleteIfNotNeededByManifestId(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t manifestId) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, std::string{ tableName } +"_deleteifnotneeded_v1_0"); + + // Get values referenced by the manifest id. + std::vector<SQLite::rowid_t> values; + + std::ostringstream selectMappingSQL; + selectMappingSQL << "SELECT [" << valueName << "] FROM [" << tableName << s_OneToManyTable_MapTable_Suffix << "] WHERE [" << s_OneToManyTable_MapTable_ManifestName << "] = ?"; + + SQLite::Statement selectMappingStatement = SQLite::Statement::Create(connection, selectMappingSQL.str()); + + selectMappingStatement.Bind(1, manifestId); + + while (selectMappingStatement.Step()) + { + values.push_back(selectMappingStatement.GetColumn<SQLite::rowid_t>(0)); + } + + // Delete the mapping table rows with the manifest id. + std::ostringstream deleteSQL; + deleteSQL << "DELETE FROM [" << tableName << s_OneToManyTable_MapTable_Suffix << "] WHERE [" << s_OneToManyTable_MapTable_ManifestName << "] = ?"; + + SQLite::Statement deleteStatement = SQLite::Statement::Create(connection, deleteSQL.str()); + + deleteStatement.Bind(1, manifestId); + + deleteStatement.Execute(); + + // For each value, see if any references exist + std::ostringstream selectValueMappingSQL; + selectValueMappingSQL << "SELECT [" << s_OneToManyTable_MapTable_ManifestName << "] " + << "FROM [" << tableName << s_OneToManyTable_MapTable_Suffix << "] WHERE [" << valueName << "] = ? LIMIT 1"; + + SQLite::Statement selectValueMappingStatement = SQLite::Statement::Create(connection, selectValueMappingSQL.str()); + + std::ostringstream deleteValueSQL; + deleteValueSQL << "DELETE FROM [" << tableName << "] WHERE [" << SQLite::RowIDName << "] = ?"; + + SQLite::Statement deleteValueStatement = SQLite::Statement::Create(connection, deleteValueSQL.str()); + + for (SQLite::rowid_t value : values) + { + selectValueMappingStatement.Reset(); + selectValueMappingStatement.Bind(1, value); + + // If no rows are found, we can delete the data. + if (!selectValueMappingStatement.Step()) + { + deleteValueStatement.Reset(); + deleteValueStatement.Bind(1, value); + + deleteValueStatement.Execute(); + } + } + + savepoint.Commit(); + } + + bool OneToManyTableIsEmpty(SQLite::Connection& connection, std::string_view tableName) + { + std::ostringstream countSQL; + countSQL << "SELECT COUNT(*) FROM [" << tableName << ']'; + + SQLite::Statement countStatement = SQLite::Statement::Create(connection, countSQL.str()); + + THROW_HR_IF(E_UNEXPECTED, !countStatement.Step()); + + std::ostringstream countMapSQL; + countMapSQL << "SELECT COUNT(*) FROM [" << tableName << s_OneToManyTable_MapTable_Suffix << ']'; + + SQLite::Statement countMapStatement = SQLite::Statement::Create(connection, countMapSQL.str()); + + THROW_HR_IF(E_UNEXPECTED, !countMapStatement.Step()); + + return ((countStatement.GetColumn<int>(0) == 0) && (countMapStatement.GetColumn<int>(0) == 0)); + } } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.h @@ -2,17 +2,31 @@ // Licensed under the MIT License. #pragma once #include "SQLiteWrapper.h" +#include <string> #include <string_view> +#include <vector> namespace AppInstaller::Repository::Microsoft::Schema::V1_0 { namespace details { + // Create the tables. void CreateOneToManyTable(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName); + + // Ensures that the value exists and inserts mapping entries. + void OneToManyTableEnsureExistsAndInsert(SQLite::Connection& connection, + std::string_view tableName, std::string_view valueName, + const std::vector<std::string>& values, SQLite::rowid_t manifestId); + + // Deletes the mapping rows for the given manifest, then removes any unused data rows. + void OneToManyTableDeleteIfNotNeededByManifestId(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t manifestId); + + // Determines if the table is empty. + bool OneToManyTableIsEmpty(SQLite::Connection& connection, std::string_view tableName); } - // A table that represents a value that is 1:N with a manifest. + // A table that represents a value that is 1:N with a primary entry. template <typename TableInfo> struct OneToManyTable { @@ -21,5 +35,23 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 { details::CreateOneToManyTable(connection, TableInfo::TableName(), TableInfo::ValueName()); } + + // Ensures that all values exist in the data table, and inserts into the mapping table for the given manifest id. + static void EnsureExistsAndInsert(SQLite::Connection& connection, const std::vector<std::string>& values, SQLite::rowid_t manifestId) + { + details::OneToManyTableEnsureExistsAndInsert(connection, TableInfo::TableName(), TableInfo::ValueName(), values, manifestId); + } + + // Deletes the mapping rows for the given manifest, then removes any unused data rows. + static void DeleteIfNotNeededByManifestId(SQLite::Connection& connection, SQLite::rowid_t manifestId) + { + details::OneToManyTableDeleteIfNotNeededByManifestId(connection, TableInfo::TableName(), TableInfo::ValueName(), manifestId); + } + + // Determines if the table is empty. + static bool IsEmpty(SQLite::Connection& connection) + { + return details::OneToManyTableIsEmpty(connection, TableInfo::TableName()); + } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.cpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include "Microsoft/Schema/1_0/OneToOneTable.h" +#include "Microsoft/Schema/1_0/ManifestTable.h" namespace AppInstaller::Repository::Microsoft::Schema::V1_0 @@ -12,11 +13,69 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 { std::ostringstream createTableSQL; createTableSQL << "CREATE TABLE [" << tableName << "](" - << '[' << valueName << "] TEXT PRIMARY KEY)"; + << '[' << valueName << "] TEXT NOT NULL PRIMARY KEY)"; SQLite::Statement createStatement = SQLite::Statement::Create(connection, createTableSQL.str()); createStatement.Execute(); } + + SQLite::rowid_t OneToOneTableEnsureExists(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, std::string_view value) + { + { + std::ostringstream selectSQL; + selectSQL << "SELECT [" << SQLite::RowIDName << "] FROM [" << tableName << "] WHERE [" << valueName << "] = ?"; + + SQLite::Statement select = SQLite::Statement::Create(connection, selectSQL.str()); + + select.Bind(1, value); + + if (select.Step()) + { + return select.GetColumn<SQLite::rowid_t>(0); + } + } + + std::ostringstream insertSQL; + insertSQL << "INSERT INTO [" << tableName << "] ([" << valueName << "]) VALUES (?)"; + + SQLite::Statement insert = SQLite::Statement::Create(connection, insertSQL.str()); + + insert.Bind(1, value); + + insert.Execute(); + + return connection.GetLastInsertRowID(); + } + + void OneToOneTableDeleteIfNotNeededById(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t id) + { + // If a manifest is found that references this id, then we are done. + if (ManifestTableSelectByValueId(connection, valueName, id)) + { + return; + } + + std::ostringstream deleteSQL; + deleteSQL << "DELETE FROM [" << tableName << "] WHERE [" << SQLite::RowIDName << "] = ?"; + + SQLite::Statement deleteStatement = SQLite::Statement::Create(connection, deleteSQL.str()); + + deleteStatement.Bind(1, id); + + deleteStatement.Execute(); + } + + bool OneToOneTableIsEmpty(SQLite::Connection& connection, std::string_view tableName) + { + std::ostringstream countSQL; + countSQL << "SELECT COUNT(*) FROM [" << tableName << ']'; + + SQLite::Statement countStatement = SQLite::Statement::Create(connection, countSQL.str()); + + THROW_HR_IF(E_UNEXPECTED, !countStatement.Step()); + + return (countStatement.GetColumn<int>(0) == 0); + } } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.h @@ -2,6 +2,7 @@ // Licensed under the MIT License. #pragma once #include "SQLiteWrapper.h" +#include <string> #include <string_view> @@ -9,22 +10,63 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 { namespace details { + // Creates the table. void CreateOneToOneTable(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName); + + // Ensures that the values exists in the table. + SQLite::rowid_t OneToOneTableEnsureExists(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, std::string_view value); + + // Removes the given row by its rowid if it is no longer referenced. + void OneToOneTableDeleteIfNotNeededById(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t id); + + // Determines if the table is empty. + bool OneToOneTableIsEmpty(SQLite::Connection& connection, std::string_view tableName); } - // A table that represents a value that is 1:1 with a manifest. + // A table that represents a value that is 1:1 with a primary entry. template <typename TableInfo> struct OneToOneTable { + // The value type. + using value_t = std::string; + + // The id type + using id_t = SQLite::rowid_t; + // Creates the table. static void Create(SQLite::Connection& connection) { details::CreateOneToOneTable(connection, TableInfo::TableName(), TableInfo::ValueName()); } - static std::string_view ValueName() + // The name of the table. + static constexpr std::string_view TableName() + { + return TableInfo::TableName(); + } + + // The value name of the table. + static constexpr std::string_view ValueName() { return TableInfo::ValueName(); } + + // Ensures that the given value exists in the table, returning the rowid. + static SQLite::rowid_t EnsureExists(SQLite::Connection& connection, std::string_view value) + { + return details::OneToOneTableEnsureExists(connection, TableInfo::TableName(), TableInfo::ValueName(), value); + } + + // Removes the given row by its rowid if it is no longer referenced. + static void DeleteIfNotNeededById(SQLite::Connection& connection, SQLite::rowid_t id) + { + return details::OneToOneTableDeleteIfNotNeededById(connection, TableInfo::TableName(), TableInfo::ValueName(), id); + } + + // Determines if the table is empty. + static bool IsEmpty(SQLite::Connection& connection) + { + return details::OneToOneTableIsEmpty(connection, TableInfo::TableName()); + } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/PathPartTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/PathPartTable.cpp @@ -8,23 +8,240 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 { using namespace std::string_view_literals; static constexpr std::string_view s_PathPartTable_Table_Name = "pathparts"sv; + static constexpr std::string_view s_PathPartTable_ParentIndex_Name = "pathparts_parentidx"sv; static constexpr std::string_view s_PathPartTable_ParentValue_Name = "parent"sv; static constexpr std::string_view s_PathPartTable_PartValue_Name = "pathpart"sv; + namespace + { + // Attempts to select a path part given the input. + // Returns an no value if none exists, or the rowid of the part if it is found. + std::optional<SQLite::rowid_t> SelectPathPart(SQLite::Connection& connection, std::optional<SQLite::rowid_t> parent, std::string_view part) + { + std::ostringstream selectPartSQL; + selectPartSQL << "SELECT [" << SQLite::RowIDName << "] " + << "FROM [" << s_PathPartTable_Table_Name << "] WHERE " + << '[' << s_PathPartTable_ParentValue_Name << "] " << (parent ? "= ?" : "IS NULL") << " AND " + << '[' << s_PathPartTable_PartValue_Name << "] = ?"; + + SQLite::Statement select = SQLite::Statement::Create(connection, selectPartSQL.str()); + + if (parent) + { + select.Bind(1, parent.value()); + select.Bind(2, part); + } + else + { + select.Bind(1, part); + } + + if (select.Step()) + { + return select.GetColumn<SQLite::rowid_t>(0); + } + else + { + return {}; + } + } + + // Inserts the given path part into the table, returning the rowid of the inserted row. + SQLite::rowid_t InsertPathPart(SQLite::Connection& connection, std::optional<SQLite::rowid_t> parent, std::string_view part) + { + THROW_HR_IF(E_INVALIDARG, part.empty()); + + std::ostringstream insertPartSQL; + insertPartSQL << "INSERT INTO [" << s_PathPartTable_Table_Name << "] (" + << '[' << s_PathPartTable_ParentValue_Name << "]," + << '[' << s_PathPartTable_PartValue_Name << "])" + << " VALUES (?, ?)"; + + SQLite::Statement insert = SQLite::Statement::Create(connection, insertPartSQL.str()); + + if (parent) + { + insert.Bind(1, parent.value()); + } + else + { + insert.Bind(1, nullptr); + } + insert.Bind(2, part); + + insert.Execute(); + + return connection.GetLastInsertRowID(); + } + + // Gets the parent of a given part by id. + // This should only be called when the part must exist, as it will throw if not found. + std::optional<SQLite::rowid_t> GetParentById(SQLite::Connection& connection, SQLite::rowid_t id) + { + std::ostringstream selectPartSQL; + selectPartSQL << "SELECT [" << s_PathPartTable_ParentValue_Name << "] FROM [" << s_PathPartTable_Table_Name << "] WHERE " + << '[' << SQLite::RowIDName << "] = ?"; + + SQLite::Statement select = SQLite::Statement::Create(connection, selectPartSQL.str()); + + select.Bind(1, id); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_INDEX_INTEGRITY_COMPROMISED, !select.Step()); + + if (!select.GetColumnIsNull(0)) + { + return select.GetColumn<SQLite::rowid_t>(0); + } + else + { + return {}; + } + } + + // Determines if any part references this one as their parent. + bool IsLeafPart(SQLite::Connection& connection, SQLite::rowid_t id) + { + std::ostringstream selectPartSQL; + selectPartSQL << "SELECT COUNT(*) FROM [" << s_PathPartTable_Table_Name << "] WHERE " + << '[' << s_PathPartTable_ParentValue_Name << "] = ?"; + + SQLite::Statement select = SQLite::Statement::Create(connection, selectPartSQL.str()); + + select.Bind(1, id); + + THROW_HR_IF(E_UNEXPECTED, !select.Step()); + + // No rows with this as a parent means it is a leaf. + return (select.GetColumn<int>(0) == 0); + } + + // Removes the given part by id. + void RemovePartById(SQLite::Connection& connection, SQLite::rowid_t id) + { + std::ostringstream deletePartSQL; + deletePartSQL << "DELETE FROM [" << s_PathPartTable_Table_Name << "] WHERE " + << '[' << SQLite::RowIDName << "] = ?"; + + SQLite::Statement deletePart = SQLite::Statement::Create(connection, deletePartSQL.str()); + + deletePart.Bind(1, id); + + deletePart.Execute(); + } + } + void PathPartTable::Create(SQLite::Connection& connection) { - std::ostringstream createTableSQL; - createTableSQL << "CREATE TABLE [" << s_PathPartTable_Table_Name << "](" - << '[' << s_PathPartTable_ParentValue_Name << "] INT64," - << '[' << s_PathPartTable_PartValue_Name << "] TEXT)"; + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "createPathParts_v1_0"); - SQLite::Statement createStatement = SQLite::Statement::Create(connection, createTableSQL.str()); + { + std::ostringstream createTableSQL; + createTableSQL << "CREATE TABLE [" << s_PathPartTable_Table_Name << "](" + << '[' << s_PathPartTable_ParentValue_Name << "] INT64," + << '[' << s_PathPartTable_PartValue_Name << "] TEXT NOT NULL," + << "PRIMARY KEY([" << s_PathPartTable_PartValue_Name << "], [" << s_PathPartTable_ParentValue_Name << "]))"; - createStatement.Execute(); + SQLite::Statement createStatement = SQLite::Statement::Create(connection, createTableSQL.str()); + + createStatement.Execute(); + } + + { + std::ostringstream createIndexSQL; + createIndexSQL << "CREATE INDEX [" << s_PathPartTable_ParentIndex_Name << "] " + << "ON [" << s_PathPartTable_Table_Name << "](" + << '[' << s_PathPartTable_ParentValue_Name << "])"; + + SQLite::Statement createStatement = SQLite::Statement::Create(connection, createIndexSQL.str()); + + createStatement.Execute(); + } + + savepoint.Commit(); } std::string_view PathPartTable::ValueName() { return s_PathPartTable_PartValue_Name; } + + std::tuple<bool, SQLite::rowid_t> PathPartTable::EnsurePathExists(SQLite::Connection& connection, const std::filesystem::path& relativePath, bool createIfNotFound) + { + THROW_HR_IF(E_INVALIDARG, !relativePath.has_relative_path()); + THROW_HR_IF(E_INVALIDARG, relativePath.has_root_path()); + + std::unique_ptr<SQLite::Savepoint> savepoint; + if (createIfNotFound) + { + savepoint = std::make_unique<SQLite::Savepoint>(SQLite::Savepoint::Create(connection, "ensurepathexists_v1_0")); + } + + bool partsAdded = false; + + std::optional<SQLite::rowid_t> parent; + for (const auto& part : relativePath) + { + std::string utf8part = part.u8string(); + std::optional<SQLite::rowid_t> current = SelectPathPart(connection, parent, utf8part); + + if (!current) + { + if (createIfNotFound) + { + partsAdded = true; + current = InsertPathPart(connection, parent, utf8part); + } + else + { + // Current part was not found, and we were told not to create. + // Return false to indicate that the path does not exist. + return {}; + } + } + + parent = current; + } + + if (savepoint) + { + savepoint->Commit(); + } + + // If we get this far, the path exists. + // If we were asked to create it, return whether we needed to or it was already present. + // If not, then true indicates that it exists. + return { (createIfNotFound ? partsAdded : true), parent.value() }; + } + + void PathPartTable::RemovePathById(SQLite::Connection& connection, SQLite::rowid_t id) + { + SQLite::rowid_t currentPartToRemove = id; + while (IsLeafPart(connection, currentPartToRemove)) + { + std::optional<SQLite::rowid_t> parent = GetParentById(connection, currentPartToRemove); + RemovePartById(connection, currentPartToRemove); + + // If parent was NULL, this was a root part and we can stop + if (!parent) + { + break; + } + else + { + currentPartToRemove = parent.value(); + } + } + } + + bool PathPartTable::IsEmpty(SQLite::Connection& connection) + { + std::ostringstream countSQL; + countSQL << "SELECT COUNT(*) FROM [" << s_PathPartTable_Table_Name << ']'; + + SQLite::Statement countStatement = SQLite::Statement::Create(connection, countSQL.str()); + + THROW_HR_IF(E_UNEXPECTED, !countStatement.Step()); + + return (countStatement.GetColumn<int>(0) == 0); + } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/PathPartTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/PathPartTable.h @@ -2,7 +2,9 @@ // Licensed under the MIT License. #pragma once #include "SQLiteWrapper.h" +#include <filesystem> #include <string_view> +#include <tuple> namespace AppInstaller::Repository::Microsoft::Schema::V1_0 @@ -15,5 +17,22 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Gets the value name. static std::string_view ValueName(); + + // Ensure that the given relative path exists within the path parts table. + // If createIfNotFound is true, the function will add the parts as needed. + // The result bool will indicate whether it was necessary to add the path (true), + // or it was already present (false). + // If createIfNotFound is false, the function will simply determine if the path is present. + // The result bool will indicate whether the path was found (true), or not (false). + // In all cases except createIfNotFound == false and result bool == false, the int64_t value + // will be valid and the rowid of the final path part in the path. + static std::tuple<bool, SQLite::rowid_t> EnsurePathExists(SQLite::Connection& connection, const std::filesystem::path& relativePath, bool createIfNotFound); + + // Removes the path that terminates at the given id. + // Will not remove a path part if it is referenced. + static void RemovePathById(SQLite::Connection& connection, SQLite::rowid_t id); + + // Determines if the table is empty. + static bool IsEmpty(SQLite::Connection& connection); }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ProtocolsTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ProtocolsTable.h @@ -12,8 +12,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 struct ProtocolsTableInfo { - inline static std::string_view TableName() { return "protocols"sv; } - inline static std::string_view ValueName() { return "protocol"sv; } + inline static constexpr std::string_view TableName() { return "protocols"sv; } + inline static constexpr std::string_view ValueName() { return "protocol"sv; } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/TagsTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/TagsTable.h @@ -12,8 +12,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 struct TagsTableInfo { - inline static std::string_view TableName() { return "tags"sv; } - inline static std::string_view ValueName() { return "tag"sv; } + inline static constexpr std::string_view TableName() { return "tags"sv; } + inline static constexpr std::string_view ValueName() { return "tag"sv; } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/VersionTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/VersionTable.h @@ -12,8 +12,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 struct VersionTableInfo { - inline static std::string_view TableName() { return "versions"sv; } - inline static std::string_view ValueName() { return "version"sv; } + inline static constexpr std::string_view TableName() { return "versions"sv; } + inline static constexpr std::string_view ValueName() { return "version"sv; } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h @@ -3,6 +3,9 @@ #pragma once #include "SQLiteWrapper.h" #include "Microsoft/Schema/Version.h" +#include "Manifest/Manifest.h" + +#include <filesystem> namespace AppInstaller::Repository::Microsoft::Schema @@ -22,6 +25,15 @@ namespace AppInstaller::Repository::Microsoft::Schema // Creates all of the version dependent tables within the database. virtual void CreateTables(SQLite::Connection& connection) = 0; + + // Adds the manifest at the repository relative path to the index. + virtual void AddManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) = 0; + + // Updates the manifest at the repository relative path in the index. + virtual bool UpdateManifest(SQLite::Connection& connection, const Manifest::Manifest& oldManifest, const std::filesystem::path& oldRelativePath, const Manifest::Manifest& newManifest, const std::filesystem::path& newRelativePath) = 0; + + // Removes the manifest at the repository relative path from the index. + virtual void RemoveManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) = 0; }; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.h @@ -14,6 +14,7 @@ namespace AppInstaller::Repository::Microsoft::Schema // Version 1.0 static constexpr std::string_view s_MetadataValueName_MajorVersion = "majorVersion"sv; static constexpr std::string_view s_MetadataValueName_MinorVersion = "minorVersion"sv; + static constexpr std::string_view s_MetadataValueName_LastWriteTime = "lastwritetime"sv; // The metadata table for the index. // Contains a fixed-schema set of named values that can be used to determine how to read the rest of the index. diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/Version.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/Version.h @@ -27,6 +27,11 @@ namespace AppInstaller::Repository::Microsoft::Schema return (MajorVersion == other.MajorVersion && MinorVersion == other.MinorVersion); } + bool operator!=(Version other) const + { + return !operator==(other); + } + // Gets a version that represents the latest schema known to the implementation. static constexpr Version Latest() { return { std::numeric_limits<uint32_t>::max(), std::numeric_limits<uint32_t>::max() }; } diff --git a/src/AppInstallerRepositoryCore/Public/AppInstallerRepositorySearch.h b/src/AppInstallerRepositoryCore/Public/AppInstallerRepositorySearch.h @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once + + +namespace AppInstaller::Repository +{ + // Container for data used to filter the available manifests in a source. + struct SearchFilter + { + + }; + + // Search result data. + struct SearchResult + { + + }; +} diff --git a/src/AppInstallerRepositoryCore/Public/AppInstallerRepositorySource.h b/src/AppInstallerRepositoryCore/Public/AppInstallerRepositorySource.h @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include <Public/AppInstallerRepositorySearch.h> + +#include <memory> +#include <string> +#include <vector> + + +namespace AppInstaller::Repository +{ + // Interface for retrieving information about a source without acting on it. + struct ISourceDetails + { + // Gets the name of the source. + virtual const std::string& GetName() = 0; + + // Gets the type of the source. + virtual const std::string& GetType() = 0; + + // Gets the argument used when adding the source. + virtual const std::string& GetArg() = 0; + + // Gets sources extra data string. + virtual const std::string& GetData() = 0; + }; + + // Interface for interacting with a source from outside of the repository lib. + struct ISource : public ISourceDetails + { + // Request that the source update its internal data from the upstream location. + virtual void Update() = 0; + + // Execute a search on the source. + virtual SearchResult Search(const SearchFilter& filter) = 0; + }; + + // Adds a new source for the user. + std::unique_ptr<ISource> AddSource(const std::string& name, const std::string& type, const std::string& arg); + + // Opens an existing source. + // Passing an empty string as the name of the source will return a source that aggregates all others. + std::unique_ptr<ISource> OpenSource(const std::string& name); + + // Gets the details for all sources. + std::vector<std::unique_ptr<ISourceDetails>> GetSources(); + + // Removes an existing source. + void RemoveSource(const std::string& name); +} diff --git a/src/AppInstallerRepositoryCore/SQLiteWrapper.cpp b/src/AppInstallerRepositoryCore/SQLiteWrapper.cpp @@ -5,6 +5,8 @@ #include <wil/result_macros.h> +using namespace std::string_view_literals; + // TODO: Invoke the wil error handling callback to log the error #define THROW_SQLITE(_error_) \ do { \ @@ -23,6 +25,8 @@ namespace AppInstaller::Repository::SQLite { + std::string_view RowIDName = "rowid"sv; + namespace { size_t GetNextStatementId() @@ -34,6 +38,11 @@ namespace AppInstaller::Repository::SQLite namespace details { + void ParameterSpecificsImpl<nullptr_t>::Bind(sqlite3_stmt* stmt, int index, nullptr_t) + { + THROW_IF_SQLITE_FAILED(sqlite3_bind_null(stmt, index)); + } + void ParameterSpecificsImpl<std::string>::Bind(sqlite3_stmt* stmt, int index, const std::string& v) { THROW_IF_SQLITE_FAILED(sqlite3_bind_text64(stmt, index, v.c_str(), v.size(), SQLITE_TRANSIENT, SQLITE_UTF8)); @@ -58,6 +67,16 @@ namespace AppInstaller::Repository::SQLite { return sqlite3_column_int(stmt, column); } + + void ParameterSpecificsImpl<int64_t>::Bind(sqlite3_stmt* stmt, int index, int64_t v) + { + THROW_IF_SQLITE_FAILED(sqlite3_bind_int64(stmt, index, v)); + } + + int64_t ParameterSpecificsImpl<int64_t>::GetColumn(sqlite3_stmt* stmt, int column) + { + return sqlite3_column_int64(stmt, column); + } } Connection::Connection(const std::string& target, OpenDisposition disposition, OpenFlags flags) @@ -81,6 +100,11 @@ namespace AppInstaller::Repository::SQLite sqlite3_close_v2(m_dbconn); } + int64_t Connection::GetLastInsertRowID() + { + return sqlite3_last_insert_rowid(m_dbconn); + } + Statement::Statement(Connection& connection, std::string_view sql, bool persistent) { m_id = GetNextStatementId(); @@ -147,6 +171,12 @@ namespace AppInstaller::Repository::SQLite THROW_HR_IF(E_UNEXPECTED, Step(failFastOnError)); } + bool Statement::GetColumnIsNull(int column) + { + int type = sqlite3_column_type(m_stmt, column); + return type == SQLITE_NULL; + } + void Statement::Reset() { AICLI_LOG(SQL, Verbose, << "Reset statement #" << m_id); @@ -160,9 +190,9 @@ namespace AppInstaller::Repository::SQLite { using namespace std::string_literals; - Statement begin = Statement::Create(connection, "SAVEPOINT ["s + name + "]"); - m_rollback = Statement::Create(connection, "ROLLBACK TO ["s + name + "]", true); - m_commit = Statement::Create(connection, "RELEASE ["s + name + "]", true); + Statement begin = Statement::Create(connection, "SAVEPOINT ["s + m_name + "]"); + m_rollback = Statement::Create(connection, "ROLLBACK TO ["s + m_name + "]", true); + m_commit = Statement::Create(connection, "RELEASE ["s + m_name + "]", true); AICLI_LOG(SQL, Info, << "Begin savepoint: " << m_name); begin.Step(); diff --git a/src/AppInstallerRepositoryCore/SQLiteWrapper.h b/src/AppInstallerRepositoryCore/SQLiteWrapper.h @@ -4,6 +4,9 @@ #include <wil/result_macros.h> #include <winsqlite/winsqlite3.h> +#include <AppInstallerLogging.h> +#include <AppInstallerLanguageUtilities.h> + #include <string> #include <string_view> #include <tuple> @@ -12,6 +15,12 @@ namespace AppInstaller::Repository::SQLite { + // The name of the rowid column in SQLite. + extern std::string_view RowIDName; + + // The type of a rowid column in code. + using rowid_t = int64_t; + namespace details { template <typename T> @@ -28,6 +37,12 @@ namespace AppInstaller::Repository::SQLite }; template <> + struct ParameterSpecificsImpl<nullptr_t> + { + static void Bind(sqlite3_stmt* stmt, int index, nullptr_t); + }; + + template <> struct ParameterSpecificsImpl<std::string> { static void Bind(sqlite3_stmt* stmt, int index, const std::string& v); @@ -47,6 +62,13 @@ namespace AppInstaller::Repository::SQLite static int GetColumn(sqlite3_stmt* stmt, int column); }; + template <> + struct ParameterSpecificsImpl<int64_t> + { + static void Bind(sqlite3_stmt* stmt, int index, int64_t v); + static int64_t GetColumn(sqlite3_stmt* stmt, int column); + }; + template <typename T> using ParameterSpecifics = ParameterSpecificsImpl<std::decay_t<T>>; } @@ -92,6 +114,8 @@ namespace AppInstaller::Repository::SQLite ~Connection(); + int64_t GetLastInsertRowID(); + operator sqlite3* () const { return m_dbconn; } private: @@ -140,6 +164,7 @@ namespace AppInstaller::Repository::SQLite template <typename Value> void Bind(int index, Value&& v) { + AICLI_LOG(SQL, Verbose, << "Binding statement #" << m_id << ": " << index << " => " << std::forward<Value>(v)); details::ParameterSpecifics<Value>::Bind(m_stmt, index, std::forward<Value>(v)); } @@ -151,6 +176,10 @@ namespace AppInstaller::Repository::SQLite // Equivalent to Step, but does not ever expect a result, throwing if one is retrieved. void Execute(bool failFastOnError = false); + // Gets a boolean value that indicates whether the specified column value is null in the current row. + // The index is 0 based. + bool GetColumnIsNull(int column); + // Gets the value of the specified column from the current row. // The index is 0 based. template <typename Value> @@ -169,6 +198,7 @@ namespace AppInstaller::Repository::SQLite } // Resets the statement state, allowing it to be evaluated again. + // Note that this does not clear data bindings. void Reset(); private: @@ -215,7 +245,7 @@ namespace AppInstaller::Repository::SQLite Savepoint(Connection& connection, std::string&& name); std::string m_name; - bool m_inProgress = true; + DestructionToken m_inProgress = true; Statement m_rollback; Statement m_commit; }; diff --git a/src/AppInstallerRepositoryCore/pch.h b/src/AppInstallerRepositoryCore/pch.h @@ -5,6 +5,7 @@ #define NOMINMAX #include <windows.h> +#include <AppInstallerErrors.h> #include <AppInstallerLogging.h> #include <AppInstallerSHA256.h> #include <yaml-cpp/yaml.h> @@ -15,8 +16,10 @@ #include <winrt/Windows.Foundation.h> +#include <filesystem> #include <initializer_list> #include <iomanip> +#include <optional> #include <string> #include <string_view> #include <sstream> diff --git a/src/AppInstallerSQLiteIndexUtil/AppInstallerSQLiteIndexUtil.h b/src/AppInstallerSQLiteIndexUtil/AppInstallerSQLiteIndexUtil.h @@ -14,6 +14,10 @@ extern "C" #define APPINSTALLER_SQLITE_INDEX_VERSION_LATEST ((UINT32)-1) + // Initializes the logging infrastructure. + APPINSTALLER_SQLITE_INDEX_API AppInstallerLoggingInit( + APPINSTALLER_SQLITE_INDEX_STRING logPath); + // Creates a new index file at filePath with the given version. APPINSTALLER_SQLITE_INDEX_API AppInstallerSQLiteIndexCreate( APPINSTALLER_SQLITE_INDEX_STRING filePath, @@ -30,21 +34,24 @@ extern "C" APPINSTALLER_SQLITE_INDEX_API AppInstallerSQLiteIndexClose( APPINSTALLER_SQLITE_INDEX_HANDLE index); - // Adds a manifest to the index. + // Adds the manifest at the repository relative path to the index. + // If the function succeeds, the manifest has been added. APPINSTALLER_SQLITE_INDEX_API AppInstallerSQLiteIndexAddManifest( APPINSTALLER_SQLITE_INDEX_HANDLE index, APPINSTALLER_SQLITE_INDEX_STRING manifestPath, APPINSTALLER_SQLITE_INDEX_STRING relativePath); - // Updates a manifest in the index. + // Updates the manifest at the repository relative path in the index. + // The out value indicates whether the index was modified by the function. APPINSTALLER_SQLITE_INDEX_API AppInstallerSQLiteIndexUpdateManifest( APPINSTALLER_SQLITE_INDEX_HANDLE index, APPINSTALLER_SQLITE_INDEX_STRING oldManifestPath, APPINSTALLER_SQLITE_INDEX_STRING oldRelativePath, APPINSTALLER_SQLITE_INDEX_STRING newManifestPath, - APPINSTALLER_SQLITE_INDEX_STRING newRelativePath); + APPINSTALLER_SQLITE_INDEX_STRING newRelativePath, + bool* indexModified); - // Removes a manifest from the index. + // Removes the manifest at the repository relative path from the index. APPINSTALLER_SQLITE_INDEX_API AppInstallerSQLiteIndexRemoveManifest( APPINSTALLER_SQLITE_INDEX_HANDLE index, APPINSTALLER_SQLITE_INDEX_STRING manifestPath, diff --git a/src/AppInstallerSQLiteIndexUtil/Exports.cpp b/src/AppInstallerSQLiteIndexUtil/Exports.cpp @@ -9,6 +9,20 @@ using namespace AppInstaller::Repository::Microsoft; extern "C" { + APPINSTALLER_SQLITE_INDEX_API AppInstallerLoggingInit(APPINSTALLER_SQLITE_INDEX_STRING logPath) try + { + THROW_HR_IF(E_INVALIDARG, !logPath); + + // Enable all logs for now. + AppInstaller::Logging::Log().EnableChannel(AppInstaller::Logging::Channel::All); + AppInstaller::Logging::Log().SetLevel(AppInstaller::Logging::Level::Verbose); + AppInstaller::Logging::AddFileLogger(logPath); + AppInstaller::Logging::EnableWilFailureTelemetry(); + + return S_OK; + } + CATCH_RETURN() + APPINSTALLER_SQLITE_INDEX_API AppInstallerSQLiteIndexCreate(APPINSTALLER_SQLITE_INDEX_STRING filePath, UINT32 majorVersion, UINT32 minorVersion, APPINSTALLER_SQLITE_INDEX_HANDLE* index) try { THROW_HR_IF(E_INVALIDARG, !filePath); @@ -50,7 +64,9 @@ extern "C" } CATCH_RETURN() - APPINSTALLER_SQLITE_INDEX_API AppInstallerSQLiteIndexAddManifest(APPINSTALLER_SQLITE_INDEX_HANDLE index, APPINSTALLER_SQLITE_INDEX_STRING manifestPath, APPINSTALLER_SQLITE_INDEX_STRING relativePath) try + APPINSTALLER_SQLITE_INDEX_API AppInstallerSQLiteIndexAddManifest( + APPINSTALLER_SQLITE_INDEX_HANDLE index, + APPINSTALLER_SQLITE_INDEX_STRING manifestPath, APPINSTALLER_SQLITE_INDEX_STRING relativePath) try { THROW_HR_IF(E_INVALIDARG, !index); THROW_HR_IF(E_INVALIDARG, !manifestPath); @@ -65,7 +81,8 @@ extern "C" APPINSTALLER_SQLITE_INDEX_API AppInstallerSQLiteIndexUpdateManifest( APPINSTALLER_SQLITE_INDEX_HANDLE index, APPINSTALLER_SQLITE_INDEX_STRING oldManifestPath, APPINSTALLER_SQLITE_INDEX_STRING oldRelativePath, - APPINSTALLER_SQLITE_INDEX_STRING newManifestPath, APPINSTALLER_SQLITE_INDEX_STRING newRelativePath) try + APPINSTALLER_SQLITE_INDEX_STRING newManifestPath, APPINSTALLER_SQLITE_INDEX_STRING newRelativePath, + bool* indexModified) try { THROW_HR_IF(E_INVALIDARG, !index); THROW_HR_IF(E_INVALIDARG, !oldManifestPath); @@ -73,13 +90,19 @@ extern "C" THROW_HR_IF(E_INVALIDARG, !newManifestPath); THROW_HR_IF(E_INVALIDARG, !newRelativePath); - reinterpret_cast<SQLiteIndex*>(index)->UpdateManifest(oldManifestPath, oldRelativePath, newManifestPath, newRelativePath); + bool result = reinterpret_cast<SQLiteIndex*>(index)->UpdateManifest(oldManifestPath, oldRelativePath, newManifestPath, newRelativePath); + if (indexModified) + { + *indexModified = result; + } return S_OK; } CATCH_RETURN() - APPINSTALLER_SQLITE_INDEX_API AppInstallerSQLiteIndexRemoveManifest(APPINSTALLER_SQLITE_INDEX_HANDLE index, APPINSTALLER_SQLITE_INDEX_STRING manifestPath, APPINSTALLER_SQLITE_INDEX_STRING relativePath) try + APPINSTALLER_SQLITE_INDEX_API AppInstallerSQLiteIndexRemoveManifest( + APPINSTALLER_SQLITE_INDEX_HANDLE index, + APPINSTALLER_SQLITE_INDEX_STRING manifestPath, APPINSTALLER_SQLITE_INDEX_STRING relativePath) try { THROW_HR_IF(E_INVALIDARG, !index); THROW_HR_IF(E_INVALIDARG, !manifestPath); diff --git a/src/AppInstallerSQLiteIndexUtil/pch.h b/src/AppInstallerSQLiteIndexUtil/pch.h @@ -6,6 +6,8 @@ #include <Windows.h> #include <Public/AppInstallerStrings.h> +#include <Public/AppInstallerLogging.h> +#include <Public/AppInstallerTelemetry.h> #include <Microsoft/SQLiteIndex.h> #include <wil/result_macros.h>