commit 38bc61c81ef1dce51e84bed334e7c83e71c0e847 parent 9ad1353d976013d4108b98a2e31cb33e68981b1c Author: JohnMcPMS <johnmcp@microsoft.com> Date: Thu, 5 Mar 2020 15:10:47 -0800 Create SQLiteIndexSource with bare minimum search functionality (#46) Diffstat:
39 files changed, 1084 insertions(+), 70 deletions(-)
diff --git a/src/AppInstallerCLI/AppInstallerCLI.vcxproj b/src/AppInstallerCLI/AppInstallerCLI.vcxproj @@ -133,7 +133,7 @@ <Link> <SubSystem>Console</SubSystem> <GenerateWindowsMetadata>false</GenerateWindowsMetadata> - <AdditionalDependencies Condition="'$(Configuration)'=='Debug'">wininet.lib;shell32.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)'=='Debug'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> </ItemDefinitionGroup> <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'"> @@ -159,7 +159,7 @@ <EnableCOMDATFolding>true</EnableCOMDATFolding> <OptimizeReferences>true</OptimizeReferences> <GenerateWindowsMetadata>false</GenerateWindowsMetadata> - <AdditionalDependencies Condition="'$(Configuration)'=='Release'">wininet.lib;shell32.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)'=='Release'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> </ItemDefinitionGroup> <ItemGroup> diff --git a/src/AppInstallerCLICore/Commands/SearchCommand.cpp b/src/AppInstallerCLICore/Commands/SearchCommand.cpp @@ -13,7 +13,7 @@ namespace AppInstaller::CLI std::vector<Argument> SearchCommand::GetArguments() const { return { - Argument{ ARG_QUERY, LOCME("The query used to search for an app"), ArgumentType::Positional, true }, + Argument{ ARG_QUERY, LOCME("The query used to search for an app"), ArgumentType::Positional, false }, Argument{ ARG_ID, LOCME("If specified, filter the results by id"), ArgumentType::Standard }, Argument{ ARG_NAME, LOCME("If specified, filter the results by name"), ArgumentType::Standard }, Argument{ ARG_MONIKER, LOCME("If specified, filter the results by app moniker"), ArgumentType::Standard }, diff --git a/src/AppInstallerCLICore/Workflows/WorkflowBase.cpp b/src/AppInstallerCLICore/Workflows/WorkflowBase.cpp @@ -105,7 +105,9 @@ namespace AppInstaller::Workflow if (match.MatchCriteria.Field != ApplicationMatchField::Id && match.MatchCriteria.Field != ApplicationMatchField::Name) { - msg += ", [" + ApplicationMatchFieldToString(match.MatchCriteria.Field) + ": " + match.MatchCriteria.Value + "]"; + msg += ", ["; + msg += ApplicationMatchFieldToString(match.MatchCriteria.Field); + msg += ": " + match.MatchCriteria.Value + "]"; } m_reporter.ShowMsg(WorkflowReporter::Level::Info, msg); diff --git a/src/AppInstallerCLICore/Workflows/WorkflowBase.h b/src/AppInstallerCLICore/Workflows/WorkflowBase.h @@ -26,7 +26,7 @@ namespace AppInstaller::Workflow void ReportSearchResult(); - std::unique_ptr<AppInstaller::Repository::ISource> m_source; + std::shared_ptr<AppInstaller::Repository::ISource> m_source; AppInstaller::Repository::SearchResult m_searchResult; }; } \ No newline at end of file diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj @@ -154,6 +154,7 @@ <ClCompile Include="Downloader.cpp" /> <ClCompile Include="MsixInfo.cpp" /> <ClCompile Include="PreIndexedPackageSource.cpp" /> + <ClCompile Include="SQLiteIndexSource.cpp" /> <ClCompile Include="WorkFlow.cpp" /> <ClCompile Include="LanguageUtilities.cpp" /> <ClCompile Include="main.cpp"> diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters @@ -71,6 +71,9 @@ <ClCompile Include="PreIndexedPackageSource.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="SQLiteIndexSource.cpp"> + <Filter>Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <None Include="PropertySheet.props" /> diff --git a/src/AppInstallerCLITests/SQLiteIndex.cpp b/src/AppInstallerCLITests/SQLiteIndex.cpp @@ -19,9 +19,29 @@ using namespace std::string_literals; using namespace TestCommon; using namespace AppInstaller::Manifest; +using namespace AppInstaller::Repository; using namespace AppInstaller::Repository::Microsoft; using namespace AppInstaller::Repository::SQLite; +SQLiteIndex SimpleTestSetup(const std::string& filePath, Manifest& manifest, std::string& relativePath) +{ + SQLiteIndex index = SQLiteIndex::CreateNew(filePath, Schema::Version::Latest()); + + manifest.Id = "test.id"; + manifest.Name = "Test Name"; + manifest.AppMoniker = "testmoniker"; + manifest.Version = "1.0.0"; + manifest.Channel = "test"; + manifest.Tags = { "t1", "t2" }; + manifest.Commands = { "test1", "test2" }; + + relativePath = "test/id/1.0.0.yml"; + + index.AddManifest(manifest, relativePath); + + return index; +} + TEST_CASE("SQLiteIndexCreateLatestAndReopen", "[sqliteindex]") { TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; @@ -65,18 +85,10 @@ TEST_CASE("SQLiteIndexCreateAndAddManifest", "[sqliteindex]") TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; INFO("Using temporary file named: " << tempFile.GetPath()); - SQLiteIndex index = SQLiteIndex::CreateNew(tempFile, Schema::Version::Latest()); - Manifest manifest; - manifest.Id = "test.id"; - manifest.Name = "Test Name"; - manifest.AppMoniker = "testmoniker"; - manifest.Version = "1.0.0"; - manifest.Channel = "test"; - manifest.Tags = { "t1", "t2" }; - manifest.Commands = { "test1", "test2" }; + std::string relativePath; - index.AddManifest(manifest, "test/id/test.id-1.0.0.yml"); + SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); } TEST_CASE("SQLiteIndexCreateAndAddManifestFile", "[sqliteindex]") @@ -463,3 +475,145 @@ TEST_CASE("SQLiteIndex_PrepareForPackaging", "[sqliteindex]") index.PrepareForPackaging(); } + +TEST_CASE("SQLiteIndex_Search_IdExactMatch", "[sqliteindex]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest manifest; + std::string relativePath; + SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + + SearchRequest request; + request.Query = RequestMatch(MatchType::Exact, manifest.Id); + + auto results = index.Search(request); + REQUIRE(results.size() == 1); + REQUIRE(results[0].second.Field == ApplicationMatchField::Id); + REQUIRE(results[0].second.Type == MatchType::Exact); + REQUIRE(results[0].second.Value == manifest.Id); +} + +TEST_CASE("SQLiteIndex_Search_MultipleMatch", "[sqliteindex]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest manifest; + std::string relativePath; + SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + + manifest.Version = "2.0.0"; + index.AddManifest(manifest, relativePath + "2"); + + SearchRequest request; + request.Query = RequestMatch(MatchType::Exact, manifest.Id); + + auto results = index.Search(request); + REQUIRE(results.size() == 1); + + auto result = index.GetVersionsById(results[0].first); + REQUIRE(result.size() == 2); +} + +TEST_CASE("SQLiteIndex_Search_NoMatch", "[sqliteindex]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest manifest; + std::string relativePath; + SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + + SearchRequest request; + request.Query = RequestMatch(MatchType::Exact, "THIS DOES NOT MATCH ANYTHING!"); + + auto results = index.Search(request); + REQUIRE(results.size() == 0); +} + +TEST_CASE("SQLiteIndex_IdString", "[sqliteindex]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest manifest; + std::string relativePath; + SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + + SearchRequest request; + request.Query = RequestMatch(MatchType::Exact, manifest.Id); + + auto results = index.Search(request); + REQUIRE(results.size() == 1); + + auto result = index.GetIdStringById(results[0].first); + REQUIRE(result.has_value()); + REQUIRE(result.value() == manifest.Id); +} + +TEST_CASE("SQLiteIndex_NameString", "[sqliteindex]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest manifest; + std::string relativePath; + SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + + SearchRequest request; + request.Query = RequestMatch(MatchType::Exact, manifest.Id); + + auto results = index.Search(request); + REQUIRE(results.size() == 1); + + auto result = index.GetNameStringById(results[0].first); + REQUIRE(result.has_value()); + REQUIRE(result.value() == manifest.Name); +} + +TEST_CASE("SQLiteIndex_PathString", "[sqliteindex]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest manifest; + std::string relativePath; + SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + + SearchRequest request; + request.Query = RequestMatch(MatchType::Exact, manifest.Id); + + auto results = index.Search(request); + REQUIRE(results.size() == 1); + + auto specificResult = index.GetPathStringByKey(results[0].first, manifest.Version, manifest.Channel); + REQUIRE(specificResult.has_value()); + REQUIRE(specificResult.value() == relativePath); + + auto latestResult = index.GetPathStringByKey(results[0].first, "", manifest.Channel); + REQUIRE(latestResult.has_value()); + REQUIRE(latestResult.value() == relativePath); +} + +TEST_CASE("SQLiteIndex_Versions", "[sqliteindex]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest manifest; + std::string relativePath; + SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + + SearchRequest request; + request.Query = RequestMatch(MatchType::Exact, manifest.Id); + + auto results = index.Search(request); + REQUIRE(results.size() == 1); + + auto result = index.GetVersionsById(results[0].first); + REQUIRE(result.size() == 1); + REQUIRE(result[0].first == manifest.Version); + REQUIRE(result[0].second == manifest.Channel); +} diff --git a/src/AppInstallerCLITests/SQLiteIndexSource.cpp b/src/AppInstallerCLITests/SQLiteIndexSource.cpp @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "TestCommon.h" +#include <Manifest/Manifest.h> +#include <Microsoft/SQLiteIndexSource.h> + +using namespace std::string_literals; +using namespace TestCommon; +using namespace AppInstaller::Manifest; +using namespace AppInstaller::Repository; +using namespace AppInstaller::Repository::Microsoft; +using namespace AppInstaller::Repository::SQLite; + +std::shared_ptr<SQLiteIndexSource> SimpleTestSetup(const std::string& filePath, SourceDetails& details, Manifest& manifest, std::string& relativePath) +{ + SQLiteIndex index = SQLiteIndex::CreateNew(filePath, Schema::Version::Latest()); + + TestDataFile testManifest("GoodManifest.yml"); + manifest = Manifest::CreateFromPath(testManifest); + + relativePath = testManifest.GetPath().filename().u8string(); + + index.AddManifest(manifest, relativePath); + + details.Name = "TestName"; + details.Type = "TestType"; + details.Arg = testManifest.GetPath().parent_path().u8string(); + details.Data = ""; + + return std::make_shared<SQLiteIndexSource>(details, std::move(index)); +} + +TEST_CASE("SQLiteIndexSource_Search_IdExactMatch", "[sqliteindexsource]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + SourceDetails details; + Manifest manifest; + std::string relativePath; + std::shared_ptr<SQLiteIndexSource> source = SimpleTestSetup(tempFile, details, manifest, relativePath); + + SearchRequest request; + request.Query = RequestMatch(MatchType::Exact, manifest.Id); + + auto results = source->Search(request); + REQUIRE(results.Matches.size() == 1); + REQUIRE(results.Matches[0].Application); + REQUIRE(results.Matches[0].MatchCriteria.Field == ApplicationMatchField::Id); + REQUIRE(results.Matches[0].MatchCriteria.Type == MatchType::Exact); + REQUIRE(results.Matches[0].MatchCriteria.Value == manifest.Id); +} + +TEST_CASE("SQLiteIndexSource_Search_NoMatch", "[sqliteindexsource]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + SourceDetails details; + Manifest manifest; + std::string relativePath; + std::shared_ptr<SQLiteIndexSource> source = SimpleTestSetup(tempFile, details, manifest, relativePath); + + SearchRequest request; + request.Query = RequestMatch(MatchType::Exact, "THIS DOES NOT MATCH ANYTHING!"); + + auto results = source->Search(request); + REQUIRE(results.Matches.size() == 0); +} + +TEST_CASE("SQLiteIndexSource_Id", "[sqliteindexsource]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + SourceDetails details; + Manifest manifest; + std::string relativePath; + std::shared_ptr<SQLiteIndexSource> source = SimpleTestSetup(tempFile, details, manifest, relativePath); + + SearchRequest request; + request.Query = RequestMatch(MatchType::Exact, manifest.Id); + + auto results = source->Search(request); + REQUIRE(results.Matches.size() == 1); + REQUIRE(results.Matches[0].Application); + IApplication* app = results.Matches[0].Application.get(); + + REQUIRE(app->GetId() == manifest.Id); +} + +TEST_CASE("SQLiteIndexSource_Name", "[sqliteindexsource]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + SourceDetails details; + Manifest manifest; + std::string relativePath; + std::shared_ptr<SQLiteIndexSource> source = SimpleTestSetup(tempFile, details, manifest, relativePath); + + SearchRequest request; + request.Query = RequestMatch(MatchType::Exact, manifest.Id); + + auto results = source->Search(request); + REQUIRE(results.Matches.size() == 1); + REQUIRE(results.Matches[0].Application); + IApplication* app = results.Matches[0].Application.get(); + + REQUIRE(app->GetName() == manifest.Name); +} + +TEST_CASE("SQLiteIndexSource_Versions", "[sqliteindexsource]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + SourceDetails details; + Manifest manifest; + std::string relativePath; + std::shared_ptr<SQLiteIndexSource> source = SimpleTestSetup(tempFile, details, manifest, relativePath); + + SearchRequest request; + request.Query = RequestMatch(MatchType::Exact, manifest.Id); + + auto results = source->Search(request); + REQUIRE(results.Matches.size() == 1); + REQUIRE(results.Matches[0].Application); + IApplication* app = results.Matches[0].Application.get(); + + auto result = app->GetVersions(); + REQUIRE(result.size() == 1); + REQUIRE(result[0].first == manifest.Version); + REQUIRE(result[0].second == manifest.Channel); +} + +TEST_CASE("SQLiteIndexSource_GetManifest", "[sqliteindexsource]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + SourceDetails details; + Manifest manifest; + std::string relativePath; + std::shared_ptr<SQLiteIndexSource> source = SimpleTestSetup(tempFile, details, manifest, relativePath); + + SearchRequest request; + request.Query = RequestMatch(MatchType::Exact, manifest.Id); + + auto results = source->Search(request); + REQUIRE(results.Matches.size() == 1); + REQUIRE(results.Matches[0].Application); + IApplication* app = results.Matches[0].Application.get(); + + auto specificResult = app->GetManifest(manifest.Version, manifest.Channel); + REQUIRE(specificResult.Id == manifest.Id); + REQUIRE(specificResult.Name == manifest.Name); + REQUIRE(specificResult.Version == manifest.Version); + REQUIRE(specificResult.Channel == manifest.Channel); + + auto latestResult = app->GetManifest("", manifest.Channel); + REQUIRE(latestResult.Id == manifest.Id); + REQUIRE(latestResult.Name == manifest.Name); + REQUIRE(latestResult.Version == manifest.Version); + REQUIRE(latestResult.Channel == manifest.Channel); +} diff --git a/src/AppInstallerCLITests/Sources.cpp b/src/AppInstallerCLITests/Sources.cpp @@ -72,9 +72,9 @@ struct TestSource : public ISource TestSource() = default; TestSource(const SourceDetails& details) : m_details(details) {} - static std::unique_ptr<ISource> Create(const SourceDetails& details) + static std::shared_ptr<ISource> Create(const SourceDetails& details) { - return std::make_unique<TestSource>(details); + return std::make_shared<TestSource>(details); } // ISource @@ -83,7 +83,7 @@ struct TestSource : public ISource return m_details; } - SearchResult Search(const SearchRequest& request) const override + SearchResult Search(const SearchRequest& request) override { UNREFERENCED_PARAMETER(request); return {}; @@ -96,7 +96,7 @@ struct TestSource : public ISource struct TestSourceFactory : public ISourceFactory { using IsInitializedFunctor = std::function<bool(const SourceDetails&)>; - using CreateFunctor = std::function<std::unique_ptr<ISource>(const SourceDetails&)>; + using CreateFunctor = std::function<std::shared_ptr<ISource>(const SourceDetails&)>; using UpdateFunctor = std::function<void(SourceDetails&)>; using RemoveFunctor = std::function<void(const SourceDetails&)>; @@ -109,7 +109,7 @@ struct TestSourceFactory : public ISourceFactory return m_isInit(details); } - std::unique_ptr<ISource> Create(const SourceDetails& details) override + std::shared_ptr<ISource> Create(const SourceDetails& details) override { return m_Create(details); } diff --git a/src/AppInstallerCLITests/WorkFlow.cpp b/src/AppInstallerCLITests/WorkFlow.cpp @@ -72,22 +72,22 @@ struct TestSource : public ISource { TestApplication(const Manifest manifest) : m_manifest(manifest) {} - Manifest GetManifest(std::string_view, std::string_view) const override + Manifest GetManifest(std::string_view, std::string_view) override { return m_manifest; } - std::string GetId() const override + std::string GetId() override { return m_manifest.Id; } - std::string GetName() const override + std::string GetName() override { return m_manifest.Name; } - std::vector<std::pair<std::string, std::string>> GetVersions() const override + std::vector<std::pair<std::string, std::string>> GetVersions() override { std::vector<std::pair<std::string, std::string>> result; result.emplace_back(std::make_pair(m_manifest.Version, m_manifest.Channel)); @@ -97,7 +97,7 @@ struct TestSource : public ISource Manifest m_manifest; }; - SearchResult Search(const SearchRequest& request) const override + SearchResult Search(const SearchRequest& request) override { SearchResult result; if (request.Query.has_value()) diff --git a/src/AppInstallerCLITests/pch.h b/src/AppInstallerCLITests/pch.h @@ -21,6 +21,7 @@ #include <functional> #include <future> #include <iostream> +#include <memory> #include <sstream> #include <utility> #include <vector> diff --git a/src/AppInstallerCommonCore/Downloader.cpp b/src/AppInstallerCommonCore/Downloader.cpp @@ -11,16 +11,15 @@ using namespace AppInstaller::Runtime; namespace AppInstaller::Utility { - std::optional<std::vector<BYTE>> Download( + std::optional<std::vector<BYTE>> DownloadToStream( const std::string& url, - const std::filesystem::path& dest, + std::ostream& dest, IProgressCallback& progress, bool computeHash) { THROW_HR_IF(E_INVALIDARG, url.empty()); - THROW_HR_IF(E_INVALIDARG, dest.empty()); - AICLI_LOG(CLI, Info, << "Downloading url: " << url << " , dest: " << dest); + AICLI_LOG(CLI, Info, << "Downloading from url: " << url); wil::unique_hinternet session(InternetOpenA( "appinstaller-cli", @@ -69,8 +68,6 @@ namespace AppInstaller::Utility nullptr); AICLI_LOG(CLI, Verbose, << "Download size: " << contentLength); - std::ofstream outfile(dest, std::ofstream::binary); - // Setup hash engine SHA256 hashEngine; std::string contentHash; @@ -99,7 +96,7 @@ namespace AppInstaller::Utility hashEngine.Add(buffer.get(), bytesRead); } - outfile.write((char*)buffer.get(), bytesRead); + dest.write((char*)buffer.get(), bytesRead); bytesDownloaded += bytesRead; @@ -110,7 +107,7 @@ namespace AppInstaller::Utility } while (bytesRead != 0); - outfile.flush(); + dest.flush(); std::vector<BYTE> result; if (computeHash) @@ -124,6 +121,23 @@ namespace AppInstaller::Utility return result; } + std::optional<std::vector<BYTE>> Download( + const std::string& url, + const std::filesystem::path& dest, + IProgressCallback& progress, + bool computeHash) + { + THROW_HR_IF(E_INVALIDARG, url.empty()); + THROW_HR_IF(E_INVALIDARG, dest.empty()); + + AICLI_LOG(CLI, Info, << "Downloading to path: " << dest); + + std::filesystem::create_directories(dest.parent_path()); + std::ofstream outfile(dest, std::ofstream::binary); + + return DownloadToStream(url, outfile, progress, computeHash); + } + bool IsUrlRemote(std::string_view url) { using namespace std::string_view_literals; diff --git a/src/AppInstallerCommonCore/Public/AppInstallerDownloader.h b/src/AppInstallerCommonCore/Public/AppInstallerDownloader.h @@ -5,6 +5,7 @@ #include <filesystem> #include <optional> +#include <ostream> #include <string> #include <string_view> #include <vector> @@ -13,6 +14,16 @@ namespace AppInstaller::Utility { // Downloads a file from the given URL and places it in the given location. // url: The url to be downloaded from. http->https redirection is allowed. + // dest: The stream to be downloaded to. + // computeHash: Optional. Indicates if SHA256 hash should be calculated when downloading. + std::optional<std::vector<BYTE>> DownloadToStream( + const std::string& url, + std::ostream& dest, + IProgressCallback& progress, + bool computeHash = false); + + // Downloads a file from the given URL and places it in the given location. + // url: The url to be downloaded from. http->https redirection is allowed. // dest: The path to local file to be downloaded to. // computeHash: Optional. Indicates if SHA256 hash should be calculated when downloading. std::optional<std::vector<BYTE>> Download( diff --git a/src/AppInstallerCommonCore/Public/AppInstallerErrors.h b/src/AppInstallerCommonCore/Public/AppInstallerErrors.h @@ -19,3 +19,4 @@ #define APPINSTALLER_CLI_ERROR_SOURCE_NAME_ALREADY_EXISTS ((HRESULT)0x8A15000C) #define APPINSTALLER_CLI_ERROR_INVALID_SOURCE_TYPE ((HRESULT)0x8A15000D) #define APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE ((HRESULT)0x8A15000E) +#define APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING ((HRESULT)0x8A15000F) diff --git a/src/AppInstallerCommonCore/Public/AppInstallerSynchronization.h b/src/AppInstallerCommonCore/Public/AppInstallerSynchronization.h @@ -18,6 +18,9 @@ namespace AppInstaller::Synchronization // - No upgrade from reader to writer. struct CrossProcessReaderWriteLock { + // Create unheld lock. + CrossProcessReaderWriteLock() = default; + ~CrossProcessReaderWriteLock(); CrossProcessReaderWriteLock(const CrossProcessReaderWriteLock&) = delete; diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -188,6 +188,7 @@ <ClInclude Include="Microsoft\Schema\MetadataTable.h" /> <ClInclude Include="Microsoft\Schema\Version.h" /> <ClInclude Include="Microsoft\SQLiteIndex.h" /> + <ClInclude Include="Microsoft\SQLiteIndexSource.h" /> <ClInclude Include="pch.h" /> <ClInclude Include="SourceFactory.h" /> <ClInclude Include="SQLiteStatementBuilder.h" /> @@ -208,6 +209,7 @@ <ClCompile Include="Microsoft\Schema\MetadataTable.cpp" /> <ClCompile Include="Microsoft\Schema\Version.cpp" /> <ClCompile Include="Microsoft\SQLiteIndex.cpp" /> + <ClCompile Include="Microsoft\SQLiteIndexSource.cpp" /> <ClCompile Include="pch.cpp"> <PrecompiledHeader>Create</PrecompiledHeader> </ClCompile> diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -108,6 +108,9 @@ <ClInclude Include="Microsoft\PreIndexedPackageSourceFactory.h"> <Filter>Microsoft</Filter> </ClInclude> + <ClInclude Include="Microsoft\SQLiteIndexSource.h"> + <Filter>Microsoft</Filter> + </ClInclude> </ItemGroup> <ItemGroup> <ClCompile Include="pch.cpp"> @@ -158,6 +161,9 @@ <ClCompile Include="Microsoft\PreIndexedPackageSourceFactory.cpp"> <Filter>Microsoft</Filter> </ClCompile> + <ClCompile Include="Microsoft\SQLiteIndexSource.cpp"> + <Filter>Microsoft</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <None Include="PropertySheet.props" /> diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -3,6 +3,8 @@ #pragma once #include "pch.h" #include "Microsoft/PreIndexedPackageSourceFactory.h" +#include "Microsoft/SQLiteIndex.h" +#include "Microsoft/SQLiteIndexSource.h" using namespace std::string_literals; using namespace std::string_view_literals; @@ -51,7 +53,7 @@ namespace AppInstaller::Repository::Microsoft return !details.Data.empty(); } - std::unique_ptr<ISource> Create(const SourceDetails& details) override final + std::shared_ptr<ISource> Create(const SourceDetails& details) override final { THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); THROW_HR_IF(E_UNEXPECTED, !IsInitialized(details)); @@ -60,7 +62,7 @@ namespace AppInstaller::Repository::Microsoft return CreateInternal(details, std::move(lock)); } - virtual std::unique_ptr<ISource> CreateInternal(const SourceDetails& details, Synchronization::CrossProcessReaderWriteLock&& lock) = 0; + virtual std::shared_ptr<ISource> CreateInternal(const SourceDetails& details, Synchronization::CrossProcessReaderWriteLock&& lock) = 0; void Update(SourceDetails& details, IProgressCallback& progress) override final { @@ -141,11 +143,21 @@ namespace AppInstaller::Repository::Microsoft return Package{ nullptr }; } - std::unique_ptr<ISource> CreateInternal(const SourceDetails& details, Synchronization::CrossProcessReaderWriteLock&& lock) override + std::shared_ptr<ISource> CreateInternal(const SourceDetails& details, Synchronization::CrossProcessReaderWriteLock&& lock) override { - UNREFERENCED_PARAMETER(details); - UNREFERENCED_PARAMETER(lock); - THROW_HR(E_NOTIMPL); + auto optionalPackage = GetPackageFromDetails(details); + if (!optionalPackage) + { + AICLI_LOG(Repo, Info, << "Package not found by family name " << details.Data); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + std::filesystem::path packageLocation = optionalPackage.InstalledLocation().Path().c_str(); + packageLocation /= s_PreIndexedPackageSourceFactory_IndexFileName; + + SQLiteIndex index = SQLiteIndex::Open(packageLocation.u8string(), SQLiteIndex::OpenDisposition::Immutable); + + return std::make_shared<SQLiteIndexSource>(details, std::move(index), std::move(lock)); } void UpdateInternal(std::string packageLocation, SourceDetails&, IProgressCallback& progress) override @@ -183,11 +195,14 @@ namespace AppInstaller::Repository::Microsoft return result; } - std::unique_ptr<ISource> CreateInternal(const SourceDetails& details, Synchronization::CrossProcessReaderWriteLock&& lock) override + std::shared_ptr<ISource> CreateInternal(const SourceDetails& details, Synchronization::CrossProcessReaderWriteLock&& lock) override { - UNREFERENCED_PARAMETER(details); - UNREFERENCED_PARAMETER(lock); - THROW_HR(E_NOTIMPL); + std::filesystem::path packageLocation = GetStatePathFromDetails(details); + packageLocation /= s_PreIndexedPackageSourceFactory_IndexFileName; + + SQLiteIndex index = SQLiteIndex::Open(packageLocation.u8string(), SQLiteIndex::OpenDisposition::Read); + + return std::make_shared<SQLiteIndexSource>(details, std::move(index), std::move(lock)); } void UpdateInternal(std::string packageLocation, SourceDetails& details, IProgressCallback& progress) override diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp @@ -197,6 +197,33 @@ namespace AppInstaller::Repository::Microsoft m_interface->PrepareForPackaging(m_dbconn); } + std::vector<std::pair<SQLiteIndex::IdType, ApplicationMatchFilter>> SQLiteIndex::Search(const SearchRequest& request) + { + AICLI_LOG(Repo, Info, << "Performing search: " << request.ToString()); + + return m_interface->Search(m_dbconn, request); + } + + std::optional<std::string> SQLiteIndex::GetIdStringById(IdType id) + { + return m_interface->GetIdStringById(m_dbconn, id); + } + + std::optional<std::string> SQLiteIndex::GetNameStringById(IdType id) + { + return m_interface->GetNameStringById(m_dbconn, id); + } + + std::optional<std::string> SQLiteIndex::GetPathStringByKey(IdType id, std::string_view version, std::string_view channel) + { + return m_interface->GetPathStringByKey(m_dbconn, id, version, channel); + } + + std::vector<std::pair<std::string, std::string>> SQLiteIndex::GetVersionsById(IdType id) + { + return m_interface->GetVersionsById(m_dbconn, id); + } + // Recording last write time based on MSDN documentation stating that time returns a POSIX epoch time and thus // should be consistent across systems. void SQLiteIndex::SetLastWriteTime() diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h @@ -2,20 +2,27 @@ // Licensed under the MIT License. #pragma once #include "SQLiteWrapper.h" -#include "Microsoft/Schema/Version.h" #include "Manifest/Manifest.h" +#include "Microsoft/Schema/Version.h" +#include "Public/AppInstallerRepositorySearch.h" +#include <AppInstallerLanguageUtilities.h> #include <chrono> #include <filesystem> #include <limits> #include <memory> #include <string> +#include <utility> +#include <vector> namespace AppInstaller::Repository::Microsoft { // Holds the connection to the database, as well as the appropriate functionality to interface with it. struct SQLiteIndex { + // An id that refers to a specific application. + using IdType = SQLite::rowid_t; + SQLiteIndex(const SQLiteIndex&) = delete; SQLiteIndex& operator=(const SQLiteIndex&) = delete; @@ -72,6 +79,22 @@ namespace AppInstaller::Repository::Microsoft // Removes data that is no longer needed for an index that is to be published. void PrepareForPackaging(); + // Performs a search based on the given criteria. + std::vector<std::pair<IdType, ApplicationMatchFilter>> Search(const SearchRequest& request); + + // Gets the Id string for the given id, if present. + std::optional<std::string> GetIdStringById(IdType id); + + // Gets the Name string for the given id, if present. + std::optional<std::string> GetNameStringById(IdType id); + + // Gets the relative path string for the given { id, version, channel }, if present. + // If version is empty, gets the value for the 'latest' version. + std::optional<std::string> GetPathStringByKey(IdType id, std::string_view version, std::string_view channel); + + // Gets all versions and channels for the given id. + std::vector<std::pair<std::string, std::string>> GetVersionsById(IdType id); + private: // Constructor used to open an existing index. SQLiteIndex(const std::string& target, SQLite::Connection::OpenDisposition disposition, SQLite::Connection::OpenFlags flags); diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/SQLiteIndexSource.h" +#include "Microsoft/PreIndexedPackageSourceFactory.h" + + +namespace AppInstaller::Repository::Microsoft +{ + namespace + { + // The IApplication impl for SQLiteIndexSource. + struct Application : public IApplication + { + Application(std::shared_ptr<SQLiteIndexSource>& source, SQLiteIndex::IdType id) : + m_id(id), m_source(source) {} + + // Inherited via IApplication + std::string GetId() override + { + return GetSource()->GetIndex().GetIdStringById(m_id).value(); + } + + std::string GetName() override + { + return GetSource()->GetIndex().GetNameStringById(m_id).value(); + } + + Manifest::Manifest GetManifest(std::string_view version, std::string_view channel) override + { + std::shared_ptr<SQLiteIndexSource> source = GetSource(); + std::string relativePath = source->GetIndex().GetPathStringByKey(m_id, version, channel).value(); + + std::string fullPath = source->GetDetails().Arg; + if (fullPath.back() != '/') + { + fullPath += '/'; + } + fullPath += relativePath; + + if (Utility::IsUrlRemote(fullPath)) + { + std::ostringstream manifestStream; + + AICLI_LOG(Repo, Info, << "Downloading manifest"); + ProgressCallback emptyCallback; + (void)Utility::DownloadToStream(fullPath, manifestStream, emptyCallback); + + std::string manifestContents = manifestStream.str(); + AICLI_LOG(Repo, Verbose, << "Manifest contents: " << manifestContents); + + return Manifest::Manifest::Create(manifestContents); + } + else + { + AICLI_LOG(Repo, Info, << "Opening manifest from local file: " << fullPath); + return Manifest::Manifest::CreateFromPath(fullPath); + } + } + + std::vector<std::pair<std::string, std::string>> GetVersions() override + { + return GetSource()->GetIndex().GetVersionsById(m_id); + } + + private: + std::shared_ptr<SQLiteIndexSource> GetSource() + { + std::shared_ptr<SQLiteIndexSource> source = m_source.lock(); + THROW_HR_IF(E_NOT_VALID_STATE, !source); + return source; + } + + std::weak_ptr<SQLiteIndexSource> m_source; + SQLiteIndex::IdType m_id; + }; + } + + SQLiteIndexSource::SQLiteIndexSource(const SourceDetails& details, SQLiteIndex&& index, Synchronization::CrossProcessReaderWriteLock&& lock) : + m_details(details), m_lock(std::move(lock)), m_index(std::move(index)) + { + } + + const SourceDetails& SQLiteIndexSource::GetDetails() const + { + return m_details; + } + + SearchResult SQLiteIndexSource::Search(const SearchRequest& request) + { + auto indexResults = m_index.Search(request); + + SearchResult result; + std::shared_ptr<SQLiteIndexSource> sharedThis = shared_from_this(); + for (auto& indexResult : indexResults) + { + result.Matches.emplace_back(std::make_unique<Application>(sharedThis, indexResult.first), std::move(indexResult.second)); + } + return result; + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.h @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "Microsoft/SQLiteIndex.h" +#include "Public/AppInstallerRepositorySource.h" +#include <AppInstallerSynchronization.h> + +#include <memory> + + +namespace AppInstaller::Repository::Microsoft +{ + // A source that holds a SQLiteIndex and lock. + struct SQLiteIndexSource : public std::enable_shared_from_this<SQLiteIndexSource>, public ISource + { + SQLiteIndexSource(const SourceDetails& details, SQLiteIndex&& index, Synchronization::CrossProcessReaderWriteLock&& lock = {}); + + SQLiteIndexSource(const SQLiteIndexSource&) = delete; + SQLiteIndexSource& operator=(const SQLiteIndexSource&) = delete; + + SQLiteIndexSource(SQLiteIndexSource&&) = default; + SQLiteIndexSource& operator=(SQLiteIndexSource&&) = default; + + ~SQLiteIndexSource() = default; + + // Get the source's details. + const SourceDetails& GetDetails() const override; + + // Execute a search on the source. + SearchResult Search(const SearchRequest& request) override; + + // Gets the index. + SQLiteIndex& GetIndex() { return m_index; } + + private: + SourceDetails m_details; + Synchronization::CrossProcessReaderWriteLock m_lock; + SQLiteIndex m_index; + }; +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.cpp @@ -55,6 +55,64 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return result; } + // Gets a manifest id by the given key values. + std::optional<SQLite::rowid_t> GetManifestIdByKey(SQLite::Connection& connection, SQLite::rowid_t id, std::string_view version = "", std::string_view channel = "") + { + std::optional<SQLite::rowid_t> channelIdOpt = ChannelTable::SelectIdByValue(connection, channel); + if (!channelIdOpt && !channel.empty()) + { + // If an empty channel was given but none was found, we will just not filter on channel. + AICLI_LOG(Repo, Info, << "Did not find a Channel { " << channel << " }"); + return {}; + } + + std::optional<SQLite::rowid_t> versionIdOpt; + + if (version.empty()) + { + std::vector<std::string> versions; + + if (channelIdOpt) + { + versions = ManifestTable::GetAllValuesByIds<VersionTable, IdTable, ChannelTable>(connection, { id, channelIdOpt.value() }); + } + else + { + versions = ManifestTable::GetAllValuesByIds<VersionTable, IdTable>(connection, { id }); + } + + if (versions.empty()) + { + AICLI_LOG(Repo, Info, << "Did not find any Versions { " << id << ", " << channel << " }"); + return {}; + } + + // TODO: Implement version sort, for now assume latest == lastest + const std::string& latestVersion = versions[versions.size() - 1]; + + versionIdOpt = VersionTable::SelectIdByValue(connection, latestVersion); + } + else + { + versionIdOpt = VersionTable::SelectIdByValue(connection, version); + } + + if (!versionIdOpt) + { + AICLI_LOG(Repo, Info, << "Did not find a Version { " << version << " }"); + return {}; + } + + if (channelIdOpt) + { + return ManifestTable::SelectByValueIds<IdTable, VersionTable, ChannelTable>(connection, { id, versionIdOpt.value(), channelIdOpt.value() }); + } + else + { + return ManifestTable::SelectByValueIds<IdTable, VersionTable>(connection, { id, versionIdOpt.value() }); + } + } + // Updates the manifest column and related table based on the given value. template <typename Table> void UpdateManifestValueById(SQLite::Connection& connection, const typename Table::value_t& value, SQLite::rowid_t manifestId) @@ -258,4 +316,88 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 builder.Vacuum(); builder.Execute(connection); } + + std::vector<std::pair<SQLite::rowid_t, ApplicationMatchFilter>> Interface::Search(SQLite::Connection& connection, const SearchRequest& request) + { + // Initial implementation handles only exact match on id, future change will implement more. + // TODO: Handle more MatchTypes + // TODO: Handle more query fields + // TODO: Handle filters + // TODO: Handle maximum count + + if (request.Query) + { + std::optional<SQLite::rowid_t> id = IdTable::SelectIdByValue(connection, request.Query->Value); + if (id) + { + return { { id.value(), ApplicationMatchFilter(ApplicationMatchField::Id, MatchType::Exact, request.Query->Value) } }; + } + else + { + return {}; + } + } + else + { + // No query, get everything + std::vector<SQLite::rowid_t> ids = IdTable::GetAllRowIds(connection); + + std::vector<std::pair<SQLite::rowid_t, ApplicationMatchFilter>> result; + for (SQLite::rowid_t id : ids) + { + result.emplace_back(std::make_pair(id, ApplicationMatchFilter(ApplicationMatchField::Id, MatchType::Wildcard, ""))); + } + return result; + } + } + + std::optional<std::string> Interface::GetIdStringById(SQLite::Connection& connection, SQLite::rowid_t id) + { + return IdTable::SelectValueById(connection, id); + } + + std::optional<std::string> Interface::GetNameStringById(SQLite::Connection& connection, SQLite::rowid_t id) + { + std::optional<SQLite::rowid_t> manifestIdOpt = GetManifestIdByKey(connection, id); + + if (!manifestIdOpt) + { + AICLI_LOG(Repo, Info, << "Did not find manifest by Id id: " << id); + return {}; + } + + auto [name] = ManifestTable::GetValuesById<NameTable>(connection, manifestIdOpt.value()); + return name; + } + + std::optional<std::string> Interface::GetPathStringByKey(SQLite::Connection& connection, SQLite::rowid_t id, std::string_view version, std::string_view channel) + { + std::optional<SQLite::rowid_t> manifestIdOpt = GetManifestIdByKey(connection, id, version, channel); + + if (!manifestIdOpt) + { + AICLI_LOG(Repo, Info, << "Did not find manifest for: " << id << ", " << version << ", " << channel); + return {}; + } + + auto [pathPartId] = ManifestTable::GetIdsById<PathPartTable>(connection, manifestIdOpt.value()); + + return PathPartTable::GetPathById(connection, pathPartId); + } + + std::vector<std::pair<std::string, std::string>> Interface::GetVersionsById(SQLite::Connection& connection, SQLite::rowid_t id) + { + auto versionsAndChannels = ManifestTable::GetAllValuesById<IdTable, VersionTable, ChannelTable>(connection, id); + + // TODO: Implement version sort, for now assume latest == lastest + std::reverse(versionsAndChannels.begin(), versionsAndChannels.end()); + + std::vector<std::pair<std::string, std::string>> result; + result.reserve(versionsAndChannels.size()); + for (auto&& vac : versionsAndChannels) + { + result.emplace_back(std::make_pair(std::move(std::get<0>(vac)), std::move(std::get<1>(vac)))); + } + return result; + } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h @@ -16,5 +16,10 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 bool UpdateManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) override; void RemoveManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::filesystem::path& relativePath) override; void PrepareForPackaging(SQLite::Connection& connection) override; + std::vector<std::pair<SQLite::rowid_t, ApplicationMatchFilter>> Search(SQLite::Connection& connection, const SearchRequest& request) override; + std::optional<std::string> GetIdStringById(SQLite::Connection& connection, SQLite::rowid_t id) override; + std::optional<std::string> GetNameStringById(SQLite::Connection& connection, SQLite::rowid_t id) override; + std::optional<std::string> GetPathStringByKey(SQLite::Connection& connection, SQLite::rowid_t id, std::string_view version, std::string_view channel) override; + std::vector<std::pair<std::string, std::string>> GetVersionsById(SQLite::Connection& connection, SQLite::rowid_t id) override; }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.cpp @@ -104,6 +104,66 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return result; } + SQLite::Statement ManifestTableGetAllValuesByIds_Statement( + SQLite::Connection& connection, + std::initializer_list<SQLite::Builder::QualifiedColumn> valueColumns, + std::initializer_list<std::string_view> idColumns, + std::initializer_list<SQLite::rowid_t> ids) + { + using QCol = SQLite::Builder::QualifiedColumn; + + THROW_HR_IF(E_INVALIDARG, idColumns.size() != ids.size()); + + SQLite::Builder::StatementBuilder builder; + builder.Select(valueColumns).From(s_ManifestTable_Table_Name); + + for (const auto& valueColumn : valueColumns) + { + builder.Join(valueColumn.Table).On(QCol{ s_ManifestTable_Table_Name, valueColumn.Column }, QCol{ valueColumn.Table, SQLite::RowIDName }); + } + + bool isFirst = true; + + for (const auto& idColumn : idColumns) + { + if (isFirst) + { + builder.Where(idColumn).Equals(SQLite::Builder::Unbound); + isFirst = false; + } + else + { + builder.And(idColumn).Equals(SQLite::Builder::Unbound); + } + } + + SQLite::Statement select = builder.Prepare(connection); + + int bindIndex = 0; + for (const auto& id : ids) + { + select.Bind(++bindIndex, id); + } + + return select; + } + + std::vector<std::string> ManifestTableGetAllValuesByIds( + SQLite::Connection& connection, + std::initializer_list<SQLite::Builder::QualifiedColumn> valueColumns, + std::initializer_list<std::string_view> idColumns, + std::initializer_list<SQLite::rowid_t> ids) + { + auto select = ManifestTableGetAllValuesByIds_Statement(connection, valueColumns, idColumns, ids); + + std::vector<std::string> result; + while (select.Step()) + { + result.emplace_back(select.GetColumn<std::string>(0)); + } + return result; + } + void ManifestTableUpdateValueIdById(SQLite::Connection& connection, std::string_view valueName, SQLite::rowid_t value, SQLite::rowid_t id) { SQLite::Builder::StatementBuilder builder; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.h @@ -6,6 +6,7 @@ #include <initializer_list> #include <optional> #include <string_view> +#include <vector> namespace AppInstaller::Repository::Microsoft::Schema::V1_0 @@ -30,6 +31,20 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 SQLite::rowid_t id, std::initializer_list<SQLite::Builder::QualifiedColumn> columns); + // Gets all values for rows that match the given ids. + SQLite::Statement ManifestTableGetAllValuesByIds_Statement( + SQLite::Connection& connection, + std::initializer_list<SQLite::Builder::QualifiedColumn> valueColumns, + std::initializer_list<std::string_view> idColumns, + std::initializer_list<SQLite::rowid_t> ids); + + // Gets all values for rows that match the given ids. + std::vector<std::string> ManifestTableGetAllValuesByIds( + SQLite::Connection& connection, + std::initializer_list<SQLite::Builder::QualifiedColumn> valueColumns, + std::initializer_list<std::string_view> idColumns, + std::initializer_list<SQLite::rowid_t> ids); + // Update the value of a single column for the manifest with the given rowid. void ManifestTableUpdateValueIdById(SQLite::Connection& connection, std::string_view valueName, SQLite::rowid_t value, SQLite::rowid_t id); } @@ -80,6 +95,26 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return details::ManifestTableGetValuesById_Statement(connection, id, { SQLite::Builder::QualifiedColumn{ Tables::TableName(), Tables::ValueName() }... }).GetRow<Tables::value_t...>(); } + // Gets the values for rows that match the given ids. + template <typename ValueTable, typename... IdTables> + static std::vector<typename ValueTable::value_t> GetAllValuesByIds(SQLite::Connection& connection, std::initializer_list<SQLite::rowid_t> ids) + { + return details::ManifestTableGetAllValuesByIds(connection, { SQLite::Builder::QualifiedColumn{ ValueTable::TableName(), ValueTable::ValueName() } }, { IdTables::ValueName()... }, ids); + } + + // Gets all values for rows that match the given id. + template <typename IdTable, typename... ValueTables> + static std::vector<std::tuple<typename ValueTables::value_t...>> GetAllValuesById(SQLite::Connection& connection, SQLite::rowid_t id) + { + auto stmt = details::ManifestTableGetAllValuesByIds_Statement(connection, { SQLite::Builder::QualifiedColumn{ ValueTables::TableName(), ValueTables::ValueName() }... }, { IdTable::ValueName() }, { id }); + std::vector<std::tuple<typename ValueTables::value_t...>> result; + while (stmt.Step()) + { + result.emplace_back(stmt.GetRow<ValueTables::value_t...>()); + } + return result; + } + // Update the value of a single column for the manifest with the given rowid. template <typename Table> static void UpdateValueIdById(SQLite::Connection& connection, SQLite::rowid_t id, SQLite::rowid_t value) diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.cpp @@ -39,6 +39,38 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } } + std::optional<std::string> OneToOneTableSelectValueById(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t id) + { + SQLite::Builder::StatementBuilder selectBuilder; + selectBuilder.Select(valueName).From(tableName).Where(SQLite::RowIDName).Equals(id); + + SQLite::Statement select = selectBuilder.Prepare(connection); + + if (select.Step()) + { + return select.GetColumn<std::string>(0); + } + else + { + return {}; + } + } + + std::vector<SQLite::rowid_t> OneToOneTableGetAllRowIds(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName) + { + SQLite::Builder::StatementBuilder selectBuilder; + selectBuilder.Select(SQLite::RowIDName).From(tableName).OrderBy(valueName); + + SQLite::Statement select = selectBuilder.Prepare(connection); + + std::vector<SQLite::rowid_t> result; + while (select.Step()) + { + result.emplace_back(select.GetColumn<SQLite::rowid_t>(0)); + } + return result; + } + SQLite::rowid_t OneToOneTableEnsureExists(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, std::string_view value) { auto selectResult = OneToOneTableSelectIdByValue(connection, tableName, valueName, value); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.h @@ -5,6 +5,7 @@ #include <optional> #include <string> #include <string_view> +#include <vector> namespace AppInstaller::Repository::Microsoft::Schema::V1_0 @@ -17,6 +18,12 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Selects the value from the table, returning the rowid if it exists. std::optional<SQLite::rowid_t> OneToOneTableSelectIdByValue(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, std::string_view value); + // Selects the value from the table, returning the rowid if it exists. + std::optional<std::string> OneToOneTableSelectValueById(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t id); + + // Gets all row ids from the table. + std::vector<SQLite::rowid_t> OneToOneTableGetAllRowIds(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName); + // Ensures that the values exists in the table. SQLite::rowid_t OneToOneTableEnsureExists(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, std::string_view value); @@ -61,6 +68,18 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return details::OneToOneTableSelectIdByValue(connection, TableInfo::TableName(), TableInfo::ValueName(), value); } + // Selects the value from the table, returning it if it exists. + static std::optional<value_t> SelectValueById(SQLite::Connection& connection, id_t id) + { + return details::OneToOneTableSelectValueById(connection, TableInfo::TableName(), TableInfo::ValueName(), id); + } + + // Gets all row ids from the table. + static std::vector<SQLite::rowid_t> GetAllRowIds(SQLite::Connection& connection) + { + return details::OneToOneTableGetAllRowIds(connection, TableInfo::TableName(), TableInfo::ValueName()); + } + // Ensures that the given value exists in the table, returning the rowid. static SQLite::rowid_t EnsureExists(SQLite::Connection& connection, std::string_view value) { diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/PathPartTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/PathPartTable.cpp @@ -170,6 +170,63 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return { (createIfNotFound ? partsAdded : true), parent.value() }; } + std::optional<std::string> PathPartTable::GetPathById(SQLite::Connection& connection, SQLite::rowid_t id) + { + SQLite::Builder::StatementBuilder builder; + builder.Select({ s_PathPartTable_ParentValue_Name, s_PathPartTable_PartValue_Name }). + From(s_PathPartTable_Table_Name).Where(SQLite::RowIDName).Equals(SQLite::Builder::Unbound); + + SQLite::Statement select = builder.Prepare(connection); + + SQLite::rowid_t currentPart = id; + std::string result; + + while (true) + { + select.Reset(); + select.Bind(1, currentPart); + + if (select.Step()) + { + std::string partValue = select.GetColumn<std::string>(1); + if (result.empty()) + { + result = partValue; + } + else + { + result = partValue + '/' + result; + } + + if (select.GetColumnIsNull(0)) + { + // If the parent of this column is null, then we have reached the relative root + break; + } + else + { + currentPart = select.GetColumn<SQLite::rowid_t>(0); + } + } + else + { + if (currentPart == id) + { + // The given id did not reference an actual path + return {}; + } + else + { + // We found a broken path + AICLI_LOG(Repo, Error, << "Path part references an invalid parent: " << currentPart); + THROW_HR(APPINSTALLER_CLI_ERROR_INDEX_INTEGRITY_COMPROMISED); + } + } + } + + return result; + } + void PathPartTable::RemovePathById(SQLite::Connection& connection, SQLite::rowid_t id) { SQLite::rowid_t currentPartToRemove = id; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/PathPartTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/PathPartTable.h @@ -3,6 +3,8 @@ #pragma once #include "SQLiteWrapper.h" #include <filesystem> +#include <optional> +#include <string> #include <string_view> #include <tuple> @@ -31,6 +33,9 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // will be valid and the rowid of the final path part in the path. static std::tuple<bool, SQLite::rowid_t> EnsurePathExists(SQLite::Connection& connection, const std::filesystem::path& relativePath, bool createIfNotFound); + // Gets the path string using the given id as the leaf. + static std::optional<std::string> GetPathById(SQLite::Connection& connection, SQLite::rowid_t id); + // Removes the path that terminates at the given id. // Will not remove a path part if it is referenced. static void RemovePathById(SQLite::Connection& connection, SQLite::rowid_t id); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h @@ -2,8 +2,9 @@ // Licensed under the MIT License. #pragma once #include "SQLiteWrapper.h" -#include "Microsoft/Schema/Version.h" #include "Manifest/Manifest.h" +#include "Microsoft/Schema/Version.h" +#include "Public/AppInstallerRepositorySearch.h" #include <filesystem> @@ -39,6 +40,22 @@ namespace AppInstaller::Repository::Microsoft::Schema // Removes data that is no longer needed for an index that is to be published. virtual void PrepareForPackaging(SQLite::Connection& connection) = 0; + + // Performs a search based on the given criteria. + virtual std::vector<std::pair<SQLite::rowid_t, ApplicationMatchFilter>> Search(SQLite::Connection& connection, const SearchRequest& request) = 0; + + // Gets the Id string for the given id, if present. + virtual std::optional<std::string> GetIdStringById(SQLite::Connection& connection, SQLite::rowid_t id) = 0; + + // Gets the Name string for the given id, if present. + virtual std::optional<std::string> GetNameStringById(SQLite::Connection& connection, SQLite::rowid_t id) = 0; + + // Gets the relative path string for the given { id, version, channel }, if present. + // If version is empty, gets the value for the 'latest' version. + virtual std::optional<std::string> GetPathStringByKey(SQLite::Connection& connection, SQLite::rowid_t id, std::string_view version, std::string_view channel) = 0; + + // Gets all versions and channels for the given id. + virtual std::vector<std::pair<std::string, std::string>> GetVersionsById(SQLite::Connection& connection, SQLite::rowid_t id) = 0; }; diff --git a/src/AppInstallerRepositoryCore/Public/AppInstallerRepositorySearch.h b/src/AppInstallerRepositoryCore/Public/AppInstallerRepositorySearch.h @@ -61,25 +61,30 @@ namespace AppInstaller::Repository // The maximum number of results to return. // The default of 0 will place no limit. size_t MaximumResults{}; + + // Returns a string summarizing the search request. + std::string ToString() const; }; // A single application result from a search. struct IApplication { // Gets the id of the application. - virtual std::string GetId() const = 0; + virtual std::string GetId() = 0; // Gets the name of the application (the latest name). - virtual std::string GetName() const = 0; + virtual std::string GetName() = 0; // Gets a manifest for this application. // An empty version implies 'latest'. // An empty channel is the 'general audience'. - virtual Manifest::Manifest GetManifest(std::string_view version, std::string_view channel) const = 0; + virtual Manifest::Manifest GetManifest(std::string_view version, std::string_view channel) = 0; // Gets all versions of this application. // The pair is <version, channel>. - virtual std::vector<std::pair<std::string, std::string>> GetVersions() const = 0; + // The versions will be returned in sorted, desceding order. + // Ex. { 4, 3, 2, 1 } + virtual std::vector<std::pair<std::string, std::string>> GetVersions() = 0; }; // A single result from the search. @@ -91,7 +96,7 @@ namespace AppInstaller::Repository // The highest order field on which the application matched the search. ApplicationMatchFilter MatchCriteria; - ResultMatch(std::unique_ptr<IApplication> a, ApplicationMatchFilter f) : Application(std::move(a)), MatchCriteria(std::move(f)) {} + ResultMatch(std::unique_ptr<IApplication>&& a, ApplicationMatchFilter f) : Application(std::move(a)), MatchCriteria(std::move(f)) {} }; // Search result data. @@ -101,22 +106,43 @@ namespace AppInstaller::Repository std::vector<ResultMatch> Matches; }; - inline std::string ApplicationMatchFieldToString(ApplicationMatchField matchField) + inline std::string_view MatchTypeToString(MatchType type) { + using namespace std::string_view_literals; + + switch (type) + { + case MatchType::Exact: + return "Exact"sv; + case MatchType::Substring: + return "Substring"sv; + case MatchType::Wildcard: + return "Wildcard"sv; + case MatchType::Fuzzy: + return "Fuzzy"sv; + } + + return "UnknownMatchType"sv; + } + + inline std::string_view ApplicationMatchFieldToString(ApplicationMatchField matchField) + { + using namespace std::string_view_literals; + switch (matchField) { case ApplicationMatchField::Command: - return "Command"; + return "Command"sv; case ApplicationMatchField::Id: - return "Id"; + return "Id"sv; case ApplicationMatchField::Moniker: - return "Moniker"; + return "Moniker"sv; case ApplicationMatchField::Name: - return "Name"; + return "Name"sv; case ApplicationMatchField::Tag: - return "Tag"; + return "Tag"sv; } - return "UnknownMatchField"; + return "UnknownMatchField"sv; } } diff --git a/src/AppInstallerRepositoryCore/Public/AppInstallerRepositorySource.h b/src/AppInstallerRepositoryCore/Public/AppInstallerRepositorySource.h @@ -41,7 +41,7 @@ namespace AppInstaller::Repository virtual const SourceDetails& GetDetails() const = 0; // Execute a search on the source. - virtual SearchResult Search(const SearchRequest& request) const = 0; + virtual SearchResult Search(const SearchRequest& request) = 0; }; // Gets the details for all sources. @@ -52,7 +52,7 @@ namespace AppInstaller::Repository // Opens an existing source. // Passing an empty string as the name of the source will return a source that aggregates all others. - std::unique_ptr<ISource> OpenSource(std::string_view name, IProgressCallback& progress); + std::shared_ptr<ISource> OpenSource(std::string_view name, IProgressCallback& progress); // Updates an existing source. // Return value indicates whether the named source was found. diff --git a/src/AppInstallerRepositoryCore/RepositorySource.cpp b/src/AppInstallerRepositoryCore/RepositorySource.cpp @@ -186,7 +186,7 @@ namespace AppInstaller::Repository return GetFactoryForType(details.Type)->IsInitialized(details); } - std::unique_ptr<ISource> CreateSourceFromDetails(const SourceDetails& details) + std::shared_ptr<ISource> CreateSourceFromDetails(const SourceDetails& details) { return GetFactoryForType(details.Type)->Create(details); } @@ -244,7 +244,7 @@ namespace AppInstaller::Repository SetSourcesToSetting(s_RepositorySettings_UserSources, currentSources); } - std::unique_ptr<ISource> OpenSource(std::string_view name, IProgressCallback& progress) + std::shared_ptr<ISource> OpenSource(std::string_view name, IProgressCallback& progress) { std::vector<SourceDetails> currentSources = GetSources(); @@ -331,6 +331,33 @@ namespace AppInstaller::Repository } } + std::string SearchRequest::ToString() const + { + std::ostringstream result; + + result << "Query:"; + if (Query) + { + result << '\'' << Query.value().Value << "'[" << MatchTypeToString(Query.value().Type) << ']'; + } + else + { + result << "[none]"; + } + + for (const auto& filter : Filters) + { + result << " Filter:" << ApplicationMatchFieldToString(filter.Field) << "='" << filter.Value << "'[" << MatchTypeToString(filter.Type) << "]"; + } + + if (MaximumResults) + { + result << " Limit:" << MaximumResults; + } + + return result.str(); + } + #ifndef AICLI_DISABLE_TEST_HOOKS void TestHook_SetSourceFactoryOverride(const std::string& type, std::function<std::unique_ptr<ISourceFactory>()>&& factory) { diff --git a/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.cpp b/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.cpp @@ -284,6 +284,18 @@ namespace AppInstaller::Repository::SQLite::Builder return *this; } + StatementBuilder& StatementBuilder::OrderBy(std::string_view column) + { + OutputColumns(m_stream, " ORDER BY ", column); + return *this; + } + + StatementBuilder& StatementBuilder::OrderBy(const QualifiedColumn& column) + { + OutputColumns(m_stream, " ORDER BY ", column); + return *this; + } + StatementBuilder& StatementBuilder::InsertInto(std::string_view table) { OutputOperationAndTable(m_stream, "INSERT INTO", table); diff --git a/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.h b/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.h @@ -191,6 +191,10 @@ namespace AppInstaller::Repository::SQLite::Builder // Set the join constraint. StatementBuilder& On(const QualifiedColumn& column1, const QualifiedColumn& column2); + // Specify the ordering to use. + StatementBuilder& OrderBy(std::string_view column); + StatementBuilder& OrderBy(const QualifiedColumn& column); + // Limits the result set to the given number of rows. StatementBuilder& Limit(size_t rowCount); diff --git a/src/AppInstallerRepositoryCore/SourceFactory.h b/src/AppInstallerRepositoryCore/SourceFactory.h @@ -18,7 +18,7 @@ namespace AppInstaller::Repository virtual bool IsInitialized(const SourceDetails& details) = 0; // Creates a source object from the given details. - virtual std::unique_ptr<ISource> Create(const SourceDetails& details) = 0; + virtual std::shared_ptr<ISource> Create(const SourceDetails& details) = 0; // Updates the source from the given details, writing back to the details any changes. virtual void Update(SourceDetails& details, IProgressCallback& progress) = 0; diff --git a/src/AppInstallerRepositoryCore/pch.h b/src/AppInstallerRepositoryCore/pch.h @@ -7,6 +7,7 @@ #include <AppInstallerDateTime.h> #include <AppInstallerDeployment.h> +#include <AppInstallerDownloader.h> #include <AppInstallerErrors.h> #include <AppInstallerLogging.h> #include <AppInstallerMsixInfo.h> @@ -23,6 +24,7 @@ #include <winrt/Windows.ApplicationModel.h> #include <winrt/Windows.Foundation.h> #include <winrt/Windows.Foundation.Collections.h> +#include <winrt/Windows.Storage.h> #include <algorithm> #include <filesystem> diff --git a/src/AppInstallerSQLiteIndexUtil/AppInstallerSQLiteIndexUtil.vcxproj b/src/AppInstallerSQLiteIndexUtil/AppInstallerSQLiteIndexUtil.vcxproj @@ -137,9 +137,9 @@ <ModuleDefinitionFile Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Source.def</ModuleDefinitionFile> <ModuleDefinitionFile Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">Source.def</ModuleDefinitionFile> <ModuleDefinitionFile Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">Source.def</ModuleDefinitionFile> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">winsqlite3.lib;%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">winsqlite3.lib;%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">winsqlite3.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> </ItemDefinitionGroup> <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'"> @@ -151,7 +151,7 @@ <Link> <SubSystem Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Windows</SubSystem> <ModuleDefinitionFile Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Source.def</ModuleDefinitionFile> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">winsqlite3.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> </ItemDefinitionGroup> <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'"> @@ -177,10 +177,10 @@ <ModuleDefinitionFile Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">Source.def</ModuleDefinitionFile> <ModuleDefinitionFile Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">Source.def</ModuleDefinitionFile> <ModuleDefinitionFile Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Source.def</ModuleDefinitionFile> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">winsqlite3.lib;%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">winsqlite3.lib;%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">winsqlite3.lib;%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|x64'">winsqlite3.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|x64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> </ItemDefinitionGroup> <ItemGroup>