commit 9f8ceea4fb8959552075d7c6f8679abc6a310306 parent 331d0ee8ff1e9c8841c34eb8ba4a38f1dd5fbbd9 Author: JohnMcPMS <johnmcp@microsoft.com> Date: Fri, 24 Jun 2022 10:59:32 -0700 Create WinGetUtil functionality for running installed package correlation (#2221) The goal of this change is to expose functionality to run the installed package correlation and gather the results. This will be used to validate and enhance manifests and allows us to use the same correlation code throughout the system. The biggest changes to existing code are: 1. Moving the JSON utility methods out of Repository and into Common 2. Moving the correlation code to be object oriented, enabling easier reuse (and allowing for more control in the future) Beyond those changes, the majority of the work is in creating the JSON (de)serialization and handling updating the metadata that has been collected. More work is still needed to: 1. Collect more information for diagnostic purposes, like the top results for diagnostics when there is no confident match. 2. Leveraging the previous metadata results to improve correlation. 3. Enable more control over weighting the correlation heuristics, enabling a difference between running on an end user system and in a "clean room" of validation. Diffstat:
56 files changed, 2969 insertions(+), 756 deletions(-)
diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt @@ -110,8 +110,8 @@ denelon depersist deque deserialize -Deserialize deserializer +deserializes deserializing dest devblogs @@ -301,6 +301,7 @@ MContext mday memset metadata +metadatas microsoft mimetype Minimatch diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt @@ -31,6 +31,7 @@ ashpatil Ashwini asm ASwitch +ASYNCRTIMP Atest ATL AType @@ -99,6 +100,7 @@ deleteifnotneeded desktopappinstaller dirs diskfull +dllimport dnld Dobbeleer dsc @@ -107,6 +109,7 @@ dvinns dw ecfr ecfrbrowse +efgh endian enr enums diff --git a/src/AppInstallerCLICore/ExecutionContextData.h b/src/AppInstallerCLICore/ExecutionContextData.h @@ -48,7 +48,7 @@ namespace AppInstaller::CLI::Execution PackagesToInstall, // On import: Sources for the imported packages Sources, - ARPSnapshot, + ARPCorrelationData, CorrelatedAppsAndFeaturesEntries, Dependencies, DependencySource, @@ -188,9 +188,9 @@ namespace AppInstaller::CLI::Execution }; template <> - struct DataMapping<Data::ARPSnapshot> + struct DataMapping<Data::ARPCorrelationData> { - using value_t = std::vector<Repository::Correlation::ARPEntrySnapshot>; + using value_t = Repository::Correlation::ARPCorrelationData; }; template <> diff --git a/src/AppInstallerCLICore/ExecutionReporter.cpp b/src/AppInstallerCLICore/ExecutionReporter.cpp @@ -139,6 +139,7 @@ namespace AppInstaller::CLI::Execution std::string response; if (!std::getline(m_in, response)) { + m_in.get(); THROW_HR(APPINSTALLER_CLI_ERROR_PROMPT_INPUT_ERROR); } diff --git a/src/AppInstallerCLICore/PackageCollection.h b/src/AppInstallerCLICore/PackageCollection.h @@ -5,7 +5,7 @@ #include "winget/RepositorySource.h" #include <winget/Manifest.h> -#include <json.h> +#include <json/json.h> #include <vector> diff --git a/src/AppInstallerCLICore/Workflows/InstallFlow.cpp b/src/AppInstallerCLICore/Workflows/InstallFlow.cpp @@ -615,55 +615,25 @@ namespace AppInstaller::CLI::Workflow if (installer && MightWriteToARP(installer->InstallerType)) { - Source arpSource = context.Reporter.ExecuteWithProgress( - [](IProgressCallback& progress) - { - Repository::Source result = Repository::Source(PredefinedSource::ARP); - result.Open(progress); - return result; - }, true); - - std::vector<std::tuple<Utility::LocIndString, Utility::LocIndString, Utility::LocIndString>> entries; - - for (const auto& entry : arpSource.Search({}).Matches) - { - auto installed = entry.Package->GetInstalledVersion(); - if (installed) - { - entries.emplace_back(std::make_tuple( - entry.Package->GetProperty(PackageProperty::Id), - installed->GetProperty(PackageVersionProperty::Version), - installed->GetProperty(PackageVersionProperty::Channel))); - } - } - - std::sort(entries.begin(), entries.end()); - - context.Add<Execution::Data::ARPSnapshot>(std::move(entries)); + Repository::Correlation::ARPCorrelationData data; + data.CapturePreInstallSnapshot(); + context.Add<Execution::Data::ARPCorrelationData>(std::move(data)); } } CATCH_LOG() void ReportARPChanges(Execution::Context& context) try { - if (!context.Contains(Execution::Data::ARPSnapshot)) + if (!context.Contains(Execution::Data::ARPCorrelationData)) { return; } const auto& manifest = context.Get<Execution::Data::Manifest>(); - const auto& arpSnapshot = context.Get<Execution::Data::ARPSnapshot>(); - - // Open the ARP source again to get the (potentially) changed ARP entries - Source arpSource = context.Reporter.ExecuteWithProgress( - [](IProgressCallback& progress) - { - Repository::Source result = Repository::Source(PredefinedSource::ARP); - result.Open(progress); - return result; - }, true); + auto& arpCorrelationData = context.Get<Execution::Data::ARPCorrelationData>(); - auto correlationResult = Correlation::FindARPEntryForNewlyInstalledPackage(manifest, arpSnapshot, arpSource); + arpCorrelationData.CapturePostInstallSnapshot(); + auto correlationResult = arpCorrelationData.CorrelateForNewlyInstalled(manifest); // Store the ARP entry found to match the package to record it in the tracking catalog later if (correlationResult.Package) diff --git a/src/AppInstallerCLITests/ARPChanges.cpp b/src/AppInstallerCLITests/ARPChanges.cpp @@ -231,7 +231,7 @@ TEST_CASE("ARPChanges_MSIX_Ignored", "[ARPChanges][workflow]") context << SnapshotARPEntries; - REQUIRE(!context.Contains(Data::ARPSnapshot)); + REQUIRE(!context.Contains(Data::ARPCorrelationData)); context << ReportARPChanges; @@ -245,9 +245,9 @@ TEST_CASE("ARPChanges_CheckSnapshot", "[ARPChanges][workflow]") context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); - auto snapshot = context.Get<Data::ARPSnapshot>(); + auto snapshot = context.Get<Data::ARPCorrelationData>().GetPreInstallSnapshot(); REQUIRE(context.EverythingResult.Matches.size() == snapshot.size()); @@ -279,7 +279,7 @@ TEST_CASE("ARPChanges_NoChange_NoMatch", "[ARPChanges][workflow]") TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context << ReportARPChanges; context.ExpectEvent(0, 0, 0); @@ -291,7 +291,7 @@ TEST_CASE("ARPChanges_NoChange_SingleMatch", "[ARPChanges][workflow]") TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddMatchResult("MatchId1", "MatchName1", "MatchPublisher1", "MatchVersion1"); @@ -305,7 +305,7 @@ TEST_CASE("ARPChanges_NoChange_MultiMatch", "[ARPChanges][workflow]") TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddMatchResult("MatchId1", "MatchName1", "MatchPublisher1", "MatchVersion1"); context.AddMatchResult("MatchId2", "MatchName2", "MatchPublisher2", "MatchVersion2"); @@ -320,7 +320,7 @@ TEST_CASE("ARPChanges_SingleChange_NoMatch", "[ARPChanges][workflow]") TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); @@ -334,7 +334,7 @@ TEST_CASE("ARPChanges_SingleChange_SingleMatch", "[ARPChanges][workflow]") TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.AddMatchResult("MatchId1", "MatchName1", "MatchPublisher1", "MatchVersion1"); @@ -349,7 +349,7 @@ TEST_CASE("ARPChanges_SingleChange_MultiMatch", "[ARPChanges][workflow]") TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.AddMatchResult("MatchId1", "MatchName1", "MatchPublisher1", "MatchVersion1"); @@ -365,7 +365,7 @@ TEST_CASE("ARPChanges_MultiChange_NoMatch", "[ARPChanges][workflow]") TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.AddEverythingResult("EverythingId2", "EverythingName2", "EverythingPublisher2", "EverythingVersion2"); @@ -380,7 +380,7 @@ TEST_CASE("ARPChanges_MultiChange_SingleMatch_NoOverlap", "[ARPChanges][workflow TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.AddEverythingResult("EverythingId2", "EverythingName2", "EverythingPublisher2", "EverythingVersion2"); @@ -396,7 +396,7 @@ TEST_CASE("ARPChanges_MultiChange_SingleMatch_Overlap", "[ARPChanges][workflow]" TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.AddEverythingResult("EverythingId2", "EverythingName2", "EverythingPublisher2", "EverythingVersion2"); @@ -412,7 +412,7 @@ TEST_CASE("ARPChanges_MultiChange_MultiMatch_NoOverlap", "[ARPChanges][workflow] TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.AddEverythingResult("EverythingId2", "EverythingName2", "EverythingPublisher2", "EverythingVersion2"); @@ -429,7 +429,7 @@ TEST_CASE("ARPChanges_MultiChange_MultiMatch_SingleOverlap", "[ARPChanges][workf TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.AddEverythingResult("EverythingId2", "EverythingName2", "EverythingPublisher2", "EverythingVersion2"); @@ -446,7 +446,7 @@ TEST_CASE("ARPChanges_MultiChange_MultiMatch_MultiOverlap", "[ARPChanges][workfl TestContext context; context << SnapshotARPEntries; - REQUIRE(context.Contains(Data::ARPSnapshot)); + REQUIRE(context.Contains(Data::ARPCorrelationData)); context.AddEverythingResult("EverythingId1", "EverythingName1", "EverythingPublisher1", "EverythingVersion1"); context.MatchResult.Matches.emplace_back(context.EverythingResult.Matches.back()); diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj @@ -111,7 +111,7 @@ <ClCompile> <Optimization>Disabled</Optimization> <PreprocessorDefinitions>_NO_ASYNCRTIMP;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</TreatWarningAsError> </ClCompile> <Link> @@ -129,7 +129,7 @@ <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'"> <ClCompile> <PreprocessorDefinitions>_NO_ASYNCRTIMP;WIN32;%(PreprocessorDefinitions)</PreprocessorDefinitions> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</TreatWarningAsError> </ClCompile> <Link> @@ -148,8 +148,8 @@ <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>_NO_ASYNCRTIMP;NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(MSBuildThisFileDirectory)..\AppInstallerCommonCore;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerRepositoryCore;$(MSBuildThisFileDirectory)..\AppInstallerCommonCore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore\Public;$(MSBuildThisFileDirectory)..\AppInstallerCLICore;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</TreatWarningAsError> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</TreatWarningAsError> </ClCompile> @@ -196,6 +196,7 @@ <ClCompile Include="GroupPolicy.cpp" /> <ClCompile Include="HashCommand.cpp" /> <ClCompile Include="HttpClientHelper.cpp" /> + <ClCompile Include="InstallerMetadataCollectionContext.cpp" /> <ClCompile Include="ManifestComparator.cpp" /> <ClCompile Include="JsonHelper.cpp" /> <ClCompile Include="MsiExecArguments.cpp" /> diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters @@ -191,6 +191,9 @@ <ClCompile Include="Correlation.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="InstallerMetadataCollectionContext.cpp"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="MsixManifest.cpp"> <Filter>Source Files</Filter> </ClCompile> diff --git a/src/AppInstallerCLITests/CustomHeader.cpp b/src/AppInstallerCLITests/CustomHeader.cpp @@ -7,7 +7,7 @@ #include "TestSource.h" #include "TestRestRequestHandler.h" #include <Rest/Schema/1_1/Interface.h> -#include <Rest/Schema/JsonHelper.h> +#include <winget/JsonUtil.h> #include <Rest/RestClient.h> #include <winget/Settings.h> @@ -79,7 +79,7 @@ TEST_CASE("RestClient_CustomHeader", "[RestSource][CustomHeader]") }})delimiter"); std::optional<std::string> customHeader = "Testing custom header"; - auto header = std::make_pair<>(CustomHeaderName, JsonHelper::GetUtilityString(customHeader.value())); + auto header = std::make_pair<>(CustomHeaderName, JSON::GetUtilityString(customHeader.value())); HttpClientHelper helper{ GetCustomHeaderVerificationHandler(web::http::status_codes::OK, sample, header) }; RestClient client = RestClient::Create(utility::conversions::to_utf8string("https://restsource.com/api"), customHeader, std::move(helper)); REQUIRE(client.GetSourceIdentifier() == "Source123"); @@ -127,7 +127,7 @@ TEST_CASE("RestSourceSearch_NoCustomHeader", "[RestSource][CustomHeader]") TEST_CASE("RestSourceSearch_CustomHeaderExceedingSize", "[RestSource][CustomHeader]") { std::string customHeader = "This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. This is a custom header that is longer than 1024 characters. "; - auto header = std::make_pair<>(CustomHeaderName, JsonHelper::GetUtilityString(customHeader)); + auto header = std::make_pair<>(CustomHeaderName, JSON::GetUtilityString(customHeader)); HttpClientHelper helper{ GetCustomHeaderVerificationHandler(web::http::status_codes::OK, sampleSearchResponse, header) }; REQUIRE_THROWS_HR(RestClient::Create(utility::conversions::to_utf8string("https://restsource.com/api"), customHeader, std::move(helper)), diff --git a/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp @@ -0,0 +1,677 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "TestCommon.h" +#include "TestSource.h" + +#include <winget/InstallerMetadataCollectionContext.h> + +using namespace AppInstaller; +using namespace AppInstaller::Repository; +using namespace AppInstaller::Repository::Correlation; +using namespace AppInstaller::Repository::Metadata; +using namespace TestCommon; + +namespace +{ + // Indicates to set minimal defaults on TestInput + struct MinimalDefaults_t {} MinimalDefaults; + + struct TestInput + { + TestInput() = default; + + TestInput(MinimalDefaults_t) + { + Version = "1.0"; + SupportedMetadataVersion = "1.0"; + SubmissionIdentifier = "1"; + InstallerHash = "ABCD"; + PackageData = std::make_optional<Manifest::Manifest>(); + PackageData->DefaultLocalization.Locale = "en-us"; + PackageData->DefaultLocalization.Add<Manifest::Localization::PackageName>("Name"); + PackageData->DefaultLocalization.Add<Manifest::Localization::Publisher>("Publisher"); + } + + TestInput(MinimalDefaults_t, const std::string& productVersion, const std::string& productCode, Manifest::InstallerTypeEnum installerType) : TestInput(MinimalDefaults) + { + CurrentMetadata = std::make_optional<ProductMetadata>(); + CurrentMetadata->SchemaVersion.Assign("1.0"); + CurrentMetadata->ProductVersionMin.Assign(productVersion); + CurrentMetadata->ProductVersionMax.Assign(productVersion); + auto& installerMetadata = CurrentMetadata->InstallerMetadataMap[InstallerHash.value()]; + installerMetadata.SubmissionIdentifier = SubmissionIdentifier.value(); + installerMetadata.AppsAndFeaturesEntries.push_back({}); + auto& entry = installerMetadata.AppsAndFeaturesEntries.back(); + entry.DisplayName = PackageData->DefaultLocalization.Get<Manifest::Localization::PackageName>(); + entry.Publisher = PackageData->DefaultLocalization.Get<Manifest::Localization::Publisher>(); + entry.DisplayVersion = productVersion; + entry.ProductCode = productCode; + entry.InstallerType = installerType; + } + + std::optional<std::string> Version; + std::optional<std::string> SupportedMetadataVersion; + std::optional<ProductMetadata> CurrentMetadata; + std::optional<web::json::value> SubmissionData; + std::optional<std::string> SubmissionIdentifier; + std::optional<std::string> InstallerHash; + // Schema 1.0 only cares about DefaultLocale and Locales + std::optional<Manifest::Manifest> PackageData; + + std::wstring ToJSON() + { + web::json::value json; + + if (Version) + { + json[L"version"] = JSON::GetStringValue(Version.value()); + } + + if (SupportedMetadataVersion) + { + json[L"supportedMetadataVersion"] = JSON::GetStringValue(SupportedMetadataVersion.value()); + } + + if (CurrentMetadata) + { + json[L"currentMetadata"] = CurrentMetadata->ToJson(Utility::Version{ "1.0" }, 0); + } + + if (SubmissionData) + { + json[L"submissionData"] = SubmissionData.value(); + } + else if (SubmissionIdentifier) + { + web::json::value submissionData; + submissionData[L"submissionIdentifier"] = JSON::GetStringValue(SubmissionIdentifier.value()); + json[L"submissionData"] = std::move(submissionData); + } + + if (InstallerHash || PackageData) + { + web::json::value packageData; + + if (InstallerHash) + { + packageData[L"installerHash"] = JSON::GetStringValue(InstallerHash.value()); + } + + if (PackageData) + { + packageData[L"DefaultLocale"] = LocaleToJSON(PackageData->DefaultLocalization); + + // TODO: Implement other locales + } + + json[L"packageData"] = std::move(packageData); + } + + return json.serialize(); + } + + private: + web::json::value LocaleToJSON(const Manifest::ManifestLocalization& localization) const + { + web::json::value locale; + + locale[L"PackageLocale"] = JSON::GetStringValue(localization.Locale); + + if (localization.Contains(Manifest::Localization::PackageName)) + { + locale[L"PackageName"] = JSON::GetStringValue(localization.Get<Manifest::Localization::PackageName>()); + } + + if (localization.Contains(Manifest::Localization::Publisher)) + { + locale[L"Publisher"] = JSON::GetStringValue(localization.Get<Manifest::Localization::Publisher>()); + } + + // TODO: Implement any other needed fields + + return locale; + } + }; + + struct TestOutput + { + TestOutput(const std::string& json) : OriginalJSON(json) + { + web::json::value input = web::json::value::parse(Utility::ConvertToUTF16(json)); + + auto versionString = JSON::GetRawStringValueFromJsonNode(input, L"version"); + if (versionString) + { + Version = std::move(versionString); + } + + auto submissionDataValue = JSON::GetJsonValueFromNode(input, L"submissionData"); + if (submissionDataValue) + { + SubmissionData = submissionDataValue.value(); + } + + auto installerHashString = JSON::GetRawStringValueFromJsonNode(input, L"installerHash"); + if (installerHashString) + { + InstallerHash = std::move(installerHashString); + } + + auto statusString = JSON::GetRawStringValueFromJsonNode(input, L"status"); + if (statusString) + { + Status = std::move(statusString); + } + + auto metadataValue = JSON::GetJsonValueFromNode(input, L"metadata"); + if (metadataValue && !metadataValue->get().is_null()) + { + Metadata = std::make_optional<ProductMetadata>(); + Metadata->FromJson(metadataValue->get()); + } + + auto diagnosticsValue = JSON::GetJsonValueFromNode(input, L"diagnostics"); + if (diagnosticsValue) + { + auto errorHRNumber = JSON::GetRawIntValueFromJsonNode(diagnosticsValue.value(), L"errorHR"); + if (errorHRNumber) + { + ErrorHR = std::move(errorHRNumber); + } + + auto errorTextString = JSON::GetRawStringValueFromJsonNode(diagnosticsValue.value(), L"errorText"); + if (errorTextString) + { + ErrorText = std::move(errorTextString); + } + } + } + + std::string OriginalJSON; + + std::optional<std::string> Version; + std::optional<web::json::value> SubmissionData; + std::optional<std::string> InstallerHash; + std::optional<std::string> Status; + std::optional<ProductMetadata> Metadata; + std::optional<HRESULT> ErrorHR; + std::optional<std::string> ErrorText; + + bool IsError() const + { + return Status && Status.value() == "Error"; + } + + bool IsSuccess() const + { + return Status && Status.value() == "Success"; + } + + bool IsLowConfidence() const + { + return Status && Status.value() == "LowConfidence"; + } + + void ValidateFieldPresence() const + { + REQUIRE(Version); + REQUIRE(SubmissionData); + REQUIRE(InstallerHash); + REQUIRE(Status); + + REQUIRE(IsSuccess() == Metadata.has_value()); + + REQUIRE(IsError() == ErrorHR.has_value()); + REQUIRE(IsError() == ErrorText.has_value()); + } + }; + + struct TestARPCorrelationData : public ARPCorrelationData + { + ARPCorrelationResult CorrelateForNewlyInstalled(const Manifest::Manifest&) override + { + return CorrelateForNewlyInstalledResult; + } + + ARPCorrelationResult CorrelateForNewlyInstalledResult; + }; + + InstallerMetadataCollectionContext CreateTestContext(std::unique_ptr<ARPCorrelationData>&& data, TestInput& input) + { + return { std::move(data), input.ToJSON() }; + } + + InstallerMetadataCollectionContext CreateTestContext(TestInput& input) + { + return CreateTestContext(std::make_unique<TestARPCorrelationData>(), input); + } + + TestOutput GetOutput(InstallerMetadataCollectionContext& context) + { + std::ostringstream strstr; + context.Complete(strstr); + + return { strstr.str() }; + } + + TestOutput GetOutput(TestInput& input) + { + InstallerMetadataCollectionContext context = CreateTestContext(input); + return GetOutput(context); + } + + void BadInputTest(TestInput& input) + { + TestOutput output = GetOutput(input); + REQUIRE(output.IsError()); + output.ValidateFieldPresence(); + } + + ProductMetadata MakeProductMetadata(std::string_view submissionIdentifier = "Submission 1", const std::string& installerHash = "ABCD") + { + ProductMetadata result; + result.SchemaVersion.Assign("1.0"); + result.ProductVersionMin.Assign("1.0"); + result.ProductVersionMax.Assign("1.0"); + auto& installerMetadata = result.InstallerMetadataMap[installerHash]; + installerMetadata.SubmissionIdentifier = submissionIdentifier; + installerMetadata.AppsAndFeaturesEntries.push_back({}); + auto& entry = installerMetadata.AppsAndFeaturesEntries.back(); + entry.DisplayName = "Name"; + entry.Publisher = "Publisher"; + entry.DisplayVersion = "1.0"; + entry.ProductCode = "{guid}"; + entry.InstallerType = Manifest::InstallerTypeEnum::Msi; + return result; + } + + struct TestMerge + { + TestMerge() = default; + + TestMerge(MinimalDefaults_t) + { + Version = "1.0"; + Metadatas = std::make_optional<std::vector<ProductMetadata>>(); + Metadatas->emplace_back(MakeProductMetadata()); + } + + std::optional<std::string> Version; + std::optional<std::vector<ProductMetadata>> Metadatas; + + std::wstring ToJSON() + { + web::json::value json; + + if (Version) + { + json[L"version"] = JSON::GetStringValue(Version.value()); + } + + if (Metadatas) + { + web::json::value metadatasArray; + + if (Metadatas->empty()) + { + metadatasArray = web::json::value::array(); + } + else + { + size_t index = 0; + for (auto& value : Metadatas.value()) + { + metadatasArray[index++] = value.ToJson(value.SchemaVersion, 0); + } + } + + json[L"metadatas"] = std::move(metadatasArray); + } + + return json.serialize(); + } + }; +} + +TEST_CASE("MetadataCollection_MinimumInput", "[metadata_collection]") +{ + TestInput input(MinimalDefaults); + TestOutput output = GetOutput(input); + REQUIRE(!output.IsError()); + output.ValidateFieldPresence(); + + REQUIRE(output.Version.value() == input.Version.value()); + REQUIRE(output.InstallerHash.value() == input.InstallerHash.value()); +} + +TEST_CASE("MetadataCollection_SubmissionDataCopied", "[metadata_collection]") +{ + TestInput input(MinimalDefaults); + + web::json::value submissionData; + std::wstring testValueName = L"testValueName"; + std::string testValueValue = "Test value value"; + submissionData[L"submissionIdentifier"] = JSON::GetStringValue("Required identifier"); + submissionData[testValueName] = JSON::GetStringValue(testValueValue); + + input.SubmissionData = submissionData; + + TestOutput output = GetOutput(input); + REQUIRE(!output.IsError()); + output.ValidateFieldPresence(); + + REQUIRE(output.Version.value() == input.Version.value()); + REQUIRE(output.InstallerHash.value() == input.InstallerHash.value()); + auto outputValue = JSON::GetRawStringValueFromJsonNode(output.SubmissionData.value(), testValueName); + REQUIRE(outputValue); + REQUIRE(outputValue.value() == testValueValue); +} + +TEST_CASE("MetadataCollection_BadInput", "[metadata_collection]") +{ + TestInput input(MinimalDefaults); + +#define RESET_FIELD_SECTION(_field_) \ + SECTION("No " #_field_) \ + { \ + input._field_.reset(); \ + BadInputTest(input); \ + } + + RESET_FIELD_SECTION(Version); + RESET_FIELD_SECTION(SupportedMetadataVersion); + RESET_FIELD_SECTION(SubmissionIdentifier); + RESET_FIELD_SECTION(InstallerHash); + RESET_FIELD_SECTION(PackageData); + +#undef RESET_FIELD_SECTION +} + +TEST_CASE("MetadataCollection_LowConfidence", "[metadata_collection]") +{ + TestInput input(MinimalDefaults); + // The default test correlation object won't have a package correlation set + TestOutput output = GetOutput(input); + REQUIRE(output.IsLowConfidence()); + REQUIRE(!output.Metadata); + output.ValidateFieldPresence(); +} + +TEST_CASE("MetadataCollection_NewPackage", "[metadata_collection]") +{ + TestInput input(MinimalDefaults); + auto correlationData = std::make_unique<TestARPCorrelationData>(); + + Manifest::Manifest manifest; + manifest.DefaultLocalization.Add<Manifest::Localization::PackageName>("Test Package Name"); + manifest.DefaultLocalization.Add<Manifest::Localization::Publisher>("Test Publisher"); + manifest.Version = "1.2.3"; + manifest.Installers.push_back({}); + manifest.Installers[0].ProductCode = "{guid}"; + + IPackageVersion::Metadata metadata; + metadata[PackageVersionMetadata::InstalledType] = Manifest::InstallerTypeToString(Manifest::InstallerTypeEnum::Msi); + + correlationData->CorrelateForNewlyInstalledResult.Package = std::make_shared<TestPackageVersion>(manifest, metadata); + + InstallerMetadataCollectionContext context = CreateTestContext(std::move(correlationData), input); + TestOutput output = GetOutput(context); + + REQUIRE(output.IsSuccess()); + output.ValidateFieldPresence(); + + REQUIRE(output.Metadata->ProductVersionMin.ToString() == output.Metadata->ProductVersionMax.ToString()); + REQUIRE(output.Metadata->ProductVersionMin.ToString() == manifest.Version); + REQUIRE(output.Metadata->InstallerMetadataMap.size() == 1); + REQUIRE(output.Metadata->InstallerMetadataMap.count(input.InstallerHash.value()) == 1); + const auto& entry = output.Metadata->InstallerMetadataMap[input.InstallerHash.value()]; + REQUIRE(entry.SubmissionIdentifier == input.SubmissionIdentifier.value()); + REQUIRE(entry.AppsAndFeaturesEntries.size() == 1); + REQUIRE(entry.AppsAndFeaturesEntries[0].DisplayName == manifest.DefaultLocalization.Get<Manifest::Localization::PackageName>()); + REQUIRE(entry.AppsAndFeaturesEntries[0].Publisher == manifest.DefaultLocalization.Get<Manifest::Localization::Publisher>()); + REQUIRE(entry.AppsAndFeaturesEntries[0].DisplayVersion == manifest.Version); + REQUIRE(entry.AppsAndFeaturesEntries[0].ProductCode == manifest.Installers[0].ProductCode); + REQUIRE(entry.AppsAndFeaturesEntries[0].InstallerType == Manifest::InstallerTypeEnum::Msi); + REQUIRE(output.Metadata->HistoricalMetadataList.empty()); +} + +TEST_CASE("MetadataCollection_SameSubmission_SameInstaller", "[metadata_collection]") +{ + std::string version = "1.3.5"; + std::string productCode = "{guid}"; + Manifest::InstallerTypeEnum installerType = Manifest::InstallerTypeEnum::Msi; + + TestInput input(MinimalDefaults, version, productCode, installerType); + auto correlationData = std::make_unique<TestARPCorrelationData>(); + + Manifest::Manifest manifest; + manifest.DefaultLocalization.Add<Manifest::Localization::PackageName>("Different Language Name"); + // Same publisher + manifest.DefaultLocalization.Add<Manifest::Localization::Publisher>(input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries[0].Publisher); + manifest.Version = version; + manifest.Installers.push_back({}); + manifest.Installers[0].ProductCode = productCode; + + IPackageVersion::Metadata metadata; + metadata[PackageVersionMetadata::InstalledType] = Manifest::InstallerTypeToString(installerType); + + correlationData->CorrelateForNewlyInstalledResult.Package = std::make_shared<TestPackageVersion>(manifest, metadata); + + InstallerMetadataCollectionContext context = CreateTestContext(std::move(correlationData), input); + TestOutput output = GetOutput(context); + + REQUIRE(output.IsSuccess()); + output.ValidateFieldPresence(); + + REQUIRE(output.Metadata->ProductVersionMin.ToString() == output.Metadata->ProductVersionMax.ToString()); + REQUIRE(output.Metadata->ProductVersionMin.ToString() == manifest.Version); + REQUIRE(output.Metadata->InstallerMetadataMap.size() == 1); + REQUIRE(output.Metadata->InstallerMetadataMap.count(input.InstallerHash.value()) == 1); + const auto& entry = output.Metadata->InstallerMetadataMap[input.InstallerHash.value()]; + REQUIRE(entry.SubmissionIdentifier == input.SubmissionIdentifier.value()); + REQUIRE(entry.AppsAndFeaturesEntries.size() == 2); + + // One should have all values, and the other should have just a different name + // Base which one is which off of whether Publisher is set + for (const auto& featureEntry : entry.AppsAndFeaturesEntries) + { + if (featureEntry.Publisher.empty()) + { + REQUIRE(featureEntry.DisplayName == manifest.DefaultLocalization.Get<Manifest::Localization::PackageName>()); + REQUIRE(featureEntry.DisplayVersion.empty()); + REQUIRE(featureEntry.ProductCode.empty()); + REQUIRE(featureEntry.InstallerType == Manifest::InstallerTypeEnum::Unknown); + } + else + { + REQUIRE(featureEntry.DisplayName == input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries[0].DisplayName); + REQUIRE(featureEntry.Publisher == manifest.DefaultLocalization.Get<Manifest::Localization::Publisher>()); + REQUIRE(featureEntry.DisplayVersion == manifest.Version); + REQUIRE(featureEntry.ProductCode == manifest.Installers[0].ProductCode); + REQUIRE(featureEntry.InstallerType == Manifest::InstallerTypeEnum::Msi); + } + } + REQUIRE(output.Metadata->HistoricalMetadataList.empty()); +} + +TEST_CASE("MetadataCollection_SameSubmission_NewInstaller", "[metadata_collection]") +{ + std::string versionPresent = "1.3.5"; + std::string versionIncoming = "1.3.5.1"; + std::string productCodePresent = "{guid}"; + std::string productCodeIncoming = "{guid_different}"; + Manifest::InstallerTypeEnum installerType = Manifest::InstallerTypeEnum::Msi; + + TestInput input(MinimalDefaults, versionPresent, productCodePresent, installerType); + // Change the incoming hash to be new + input.InstallerHash = input.InstallerHash.value() + "_DIFFERENT"; + auto correlationData = std::make_unique<TestARPCorrelationData>(); + + Manifest::Manifest manifest; + manifest.DefaultLocalization.Add<Manifest::Localization::PackageName>("Name (but different architecture)"); + // Same publisher + manifest.DefaultLocalization.Add<Manifest::Localization::Publisher>(input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries[0].Publisher); + manifest.Version = versionIncoming; + manifest.Installers.push_back({}); + manifest.Installers[0].ProductCode = productCodeIncoming; + + IPackageVersion::Metadata metadata; + metadata[PackageVersionMetadata::InstalledType] = Manifest::InstallerTypeToString(installerType); + + correlationData->CorrelateForNewlyInstalledResult.Package = std::make_shared<TestPackageVersion>(manifest, metadata); + + InstallerMetadataCollectionContext context = CreateTestContext(std::move(correlationData), input); + TestOutput output = GetOutput(context); + + REQUIRE(output.IsSuccess()); + output.ValidateFieldPresence(); + + REQUIRE(output.Metadata->ProductVersionMin.ToString() == versionPresent); + REQUIRE(output.Metadata->ProductVersionMax.ToString() == versionIncoming); + REQUIRE(output.Metadata->InstallerMetadataMap.size() == 2); + + for (const auto& installerMetadata : output.Metadata->InstallerMetadataMap) + { + const auto& entry = installerMetadata.second; + + REQUIRE(entry.SubmissionIdentifier == input.SubmissionIdentifier.value()); + REQUIRE(entry.AppsAndFeaturesEntries.size() == 1); + + const auto& featureEntry = entry.AppsAndFeaturesEntries.front(); + REQUIRE(featureEntry.Publisher == manifest.DefaultLocalization.Get<Manifest::Localization::Publisher>()); + + if (featureEntry.ProductCode == productCodePresent) + { + REQUIRE(featureEntry.DisplayName == input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries[0].DisplayName); + REQUIRE(featureEntry.DisplayVersion == versionPresent); + REQUIRE(featureEntry.InstallerType == Manifest::InstallerTypeEnum::Msi); + } + else + { + REQUIRE(featureEntry.DisplayName == manifest.DefaultLocalization.Get<Manifest::Localization::PackageName>()); + REQUIRE(featureEntry.DisplayVersion == versionIncoming); + REQUIRE(featureEntry.InstallerType == Manifest::InstallerTypeEnum::Msi); + REQUIRE(featureEntry.ProductCode == productCodeIncoming); + } + } + + REQUIRE(output.Metadata->HistoricalMetadataList.empty()); +} + +TEST_CASE("MetadataCollection_NewSubmission", "[metadata_collection]") +{ + std::string versionPresent = "1.3.5"; + std::string versionIncoming = "1.4.0"; + std::string productCodePresent = "{guid}"; + std::string productCodeIncoming = "{guid_different}"; + Manifest::InstallerTypeEnum installerType = Manifest::InstallerTypeEnum::Msi; + + TestInput input(MinimalDefaults, versionPresent, productCodePresent, installerType); + input.SubmissionIdentifier = input.SubmissionIdentifier.value() + "_NEW"; + input.InstallerHash = input.InstallerHash.value() + "_DIFFERENT"; + auto correlationData = std::make_unique<TestARPCorrelationData>(); + + Manifest::Manifest manifest; + manifest.DefaultLocalization.Add<Manifest::Localization::PackageName>(input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries[0].DisplayName); + manifest.DefaultLocalization.Add<Manifest::Localization::Publisher>(input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries[0].Publisher); + manifest.Version = versionIncoming; + manifest.Installers.push_back({}); + manifest.Installers[0].ProductCode = productCodeIncoming; + + IPackageVersion::Metadata metadata; + metadata[PackageVersionMetadata::InstalledType] = Manifest::InstallerTypeToString(installerType); + + correlationData->CorrelateForNewlyInstalledResult.Package = std::make_shared<TestPackageVersion>(manifest, metadata); + + InstallerMetadataCollectionContext context = CreateTestContext(std::move(correlationData), input); + TestOutput output = GetOutput(context); + + REQUIRE(output.IsSuccess()); + output.ValidateFieldPresence(); + + REQUIRE(output.Metadata->ProductVersionMin.ToString() == output.Metadata->ProductVersionMax.ToString()); + REQUIRE(output.Metadata->ProductVersionMin.ToString() == manifest.Version); + REQUIRE(output.Metadata->InstallerMetadataMap.size() == 1); + REQUIRE(output.Metadata->InstallerMetadataMap.count(input.InstallerHash.value()) == 1); + const auto& entry = output.Metadata->InstallerMetadataMap[input.InstallerHash.value()]; + REQUIRE(entry.SubmissionIdentifier == input.SubmissionIdentifier.value()); + REQUIRE(entry.AppsAndFeaturesEntries.size() == 1); + REQUIRE(entry.AppsAndFeaturesEntries[0].DisplayName == manifest.DefaultLocalization.Get<Manifest::Localization::PackageName>()); + REQUIRE(entry.AppsAndFeaturesEntries[0].Publisher == manifest.DefaultLocalization.Get<Manifest::Localization::Publisher>()); + REQUIRE(entry.AppsAndFeaturesEntries[0].DisplayVersion == manifest.Version); + REQUIRE(entry.AppsAndFeaturesEntries[0].ProductCode == manifest.Installers[0].ProductCode); + REQUIRE(entry.AppsAndFeaturesEntries[0].InstallerType == Manifest::InstallerTypeEnum::Msi); + + REQUIRE(output.Metadata->HistoricalMetadataList.size() == 1); + const auto& historicalEntry = output.Metadata->HistoricalMetadataList[0]; + REQUIRE(historicalEntry.ProductVersionMin.ToString() == input.CurrentMetadata->ProductVersionMin.ToString()); + REQUIRE(historicalEntry.ProductVersionMax.ToString() == input.CurrentMetadata->ProductVersionMax.ToString()); + const auto& appsAndFeaturesEntry = input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries.front(); + REQUIRE(historicalEntry.Names.size() == 1); + REQUIRE(*historicalEntry.Names.begin() == appsAndFeaturesEntry.DisplayName); + REQUIRE(historicalEntry.ProductCodes.size() == 1); + REQUIRE(*historicalEntry.ProductCodes.begin() == appsAndFeaturesEntry.ProductCode); + REQUIRE(historicalEntry.Publishers.size() == 1); + REQUIRE(*historicalEntry.Publishers.begin() == appsAndFeaturesEntry.Publisher); +} + +TEST_CASE("MetadataCollection_Merge_Empty", "[metadata_collection]") +{ + TestMerge mergeData{ MinimalDefaults }; + mergeData.Metadatas->clear(); + + REQUIRE_THROWS_HR(InstallerMetadataCollectionContext::Merge(mergeData.ToJSON(), 0, {}), E_NOT_SET); +} + +TEST_CASE("MetadataCollection_Merge_SubmissionMismatch", "[metadata_collection]") +{ + TestMerge mergeData{ MinimalDefaults }; + mergeData.Metadatas->emplace_back(MakeProductMetadata("Submission 2")); + + REQUIRE_THROWS_HR(InstallerMetadataCollectionContext::Merge(mergeData.ToJSON(), 0, {}), E_NOT_VALID_STATE); +} + +TEST_CASE("MetadataCollection_Merge_DifferentInstallers", "[metadata_collection]") +{ + TestMerge mergeData{ MinimalDefaults }; + mergeData.Metadatas->emplace_back(MakeProductMetadata(mergeData.Metadatas->at(0).InstallerMetadataMap.begin()->second.SubmissionIdentifier, "EFGH")); + + std::wstring mergeResult = InstallerMetadataCollectionContext::Merge(mergeData.ToJSON(), 0, {}); + REQUIRE(!mergeResult.empty()); + + ProductMetadata mergeMetadata; + mergeMetadata.FromJson(web::json::value::parse(mergeResult)); + + REQUIRE(mergeMetadata.InstallerMetadataMap.size() == 2); + for (const auto& item : mergeMetadata.InstallerMetadataMap) + { + REQUIRE(item.second.AppsAndFeaturesEntries.size() == 1); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].DisplayName.empty()); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].Publisher.empty()); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].DisplayVersion.empty()); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].ProductCode.empty()); + } +} + +TEST_CASE("MetadataCollection_Merge_SameInstaller", "[metadata_collection]") +{ + TestMerge mergeData{ MinimalDefaults }; + mergeData.Metadatas->emplace_back(MakeProductMetadata()); + + std::wstring mergeResult = InstallerMetadataCollectionContext::Merge(mergeData.ToJSON(), 0, {}); + REQUIRE(!mergeResult.empty()); + + ProductMetadata mergeMetadata; + mergeMetadata.FromJson(web::json::value::parse(mergeResult)); + + REQUIRE(mergeMetadata.InstallerMetadataMap.size() == 1); + for (const auto& item : mergeMetadata.InstallerMetadataMap) + { + REQUIRE(item.second.AppsAndFeaturesEntries.size() == 1); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].DisplayName.empty()); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].Publisher.empty()); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].DisplayVersion.empty()); + REQUIRE(!item.second.AppsAndFeaturesEntries[0].ProductCode.empty()); + } +} diff --git a/src/AppInstallerCLITests/JsonHelper.cpp b/src/AppInstallerCLITests/JsonHelper.cpp @@ -2,10 +2,10 @@ // Licensed under the MIT License. #include "pch.h" #include "TestCommon.h" -#include "Rest/Schema/JsonHelper.h" +#include <winget/JsonUtil.h> #include "cpprest/json.h" -using namespace AppInstaller::Repository::Rest::Schema; +using namespace AppInstaller; web::json::value GetTestJsonObject() { @@ -25,41 +25,41 @@ web::json::value GetTestJsonObject() TEST_CASE("GetUtilityString", "[RestSource]") { - REQUIRE(JsonHelper::GetUtilityString("cpprest") == L"cpprest"); - REQUIRE(JsonHelper::GetUtilityString(" ") == L" "); + REQUIRE(JSON::GetUtilityString("cpprest") == L"cpprest"); + REQUIRE(JSON::GetUtilityString(" ") == L" "); } TEST_CASE("GetJsonValueFromNode", "[RestSource]") { web::json::value jsonObject = GetTestJsonObject(); - std::optional<std::reference_wrapper<const web::json::value>> actual = JsonHelper::GetJsonValueFromNode(jsonObject, L"Key1"); + std::optional<std::reference_wrapper<const web::json::value>> actual = JSON::GetJsonValueFromNode(jsonObject, L"Key1"); REQUIRE(actual); REQUIRE(actual.value().get().as_string() == L"Value1"); - std::optional<std::reference_wrapper<const web::json::value>> absentKey = JsonHelper::GetJsonValueFromNode(jsonObject, L"Key3"); + std::optional<std::reference_wrapper<const web::json::value>> absentKey = JSON::GetJsonValueFromNode(jsonObject, L"Key3"); REQUIRE(!absentKey); web::json::value emptyObject; - std::optional<std::reference_wrapper<const web::json::value>> empty = JsonHelper::GetJsonValueFromNode(emptyObject, L"Key1"); + std::optional<std::reference_wrapper<const web::json::value>> empty = JSON::GetJsonValueFromNode(emptyObject, L"Key1"); REQUIRE(!empty); } TEST_CASE("GetRawStringValueFromJsonValue", "[RestSource]") { - std::optional<std::string> stringTest = JsonHelper::GetRawStringValueFromJsonValue(web::json::value::string(L"cpprest ")); + std::optional<std::string> stringTest = JSON::GetRawStringValueFromJsonValue(web::json::value::string(L"cpprest ")); REQUIRE(stringTest); REQUIRE(stringTest.value() == "cpprest "); - std::optional<std::string> emptyTest = JsonHelper::GetRawStringValueFromJsonValue(web::json::value::string(L" ")); + std::optional<std::string> emptyTest = JSON::GetRawStringValueFromJsonValue(web::json::value::string(L" ")); REQUIRE(emptyTest); REQUIRE(emptyTest.value() == " "); web::json::value obj; - std::optional<std::string> nullTest = JsonHelper::GetRawStringValueFromJsonValue(obj); + std::optional<std::string> nullTest = JSON::GetRawStringValueFromJsonValue(obj); REQUIRE(!nullTest); web::json::value integer = 100; - std::optional<std::string> mismatchFieldTest = JsonHelper::GetRawStringValueFromJsonValue(integer); + std::optional<std::string> mismatchFieldTest = JSON::GetRawStringValueFromJsonValue(integer); REQUIRE(!mismatchFieldTest); } @@ -67,47 +67,47 @@ TEST_CASE("GetRawStringValueFromJsonNode", "[RestSource]") { web::json::value jsonObject = GetTestJsonObject(); - std::optional<std::string> stringTest = JsonHelper::GetRawStringValueFromJsonNode(jsonObject, L"Key1"); + std::optional<std::string> stringTest = JSON::GetRawStringValueFromJsonNode(jsonObject, L"Key1"); REQUIRE(stringTest); REQUIRE(stringTest.value() == "Value1"); - std::optional<std::string> emptyTest = JsonHelper::GetRawStringValueFromJsonNode(jsonObject, L"Key3"); + std::optional<std::string> emptyTest = JSON::GetRawStringValueFromJsonNode(jsonObject, L"Key3"); REQUIRE(!emptyTest); - std::optional<std::string> mismatchFieldTest = JsonHelper::GetRawStringValueFromJsonNode(jsonObject, L"IntKey"); + std::optional<std::string> mismatchFieldTest = JSON::GetRawStringValueFromJsonNode(jsonObject, L"IntKey"); REQUIRE(!mismatchFieldTest); } TEST_CASE("GetRawIntValueFromJsonValue", "[RestSource]") { web::json::value jsonObject = 100; - std::optional<int> expected = JsonHelper::GetRawIntValueFromJsonValue(jsonObject); + std::optional<int> expected = JSON::GetRawIntValueFromJsonValue(jsonObject); REQUIRE(expected); REQUIRE(expected.value() == 100); - std::optional<int> mismatchFieldTest = JsonHelper::GetRawIntValueFromJsonValue(web::json::value::string(L"cpprest")); + std::optional<int> mismatchFieldTest = JSON::GetRawIntValueFromJsonValue(web::json::value::string(L"cpprest")); REQUIRE(!mismatchFieldTest); } TEST_CASE("GetRawJsonArrayFromJsonNode", "[RestSource]") { web::json::value jsonObject = GetTestJsonObject(); - std::optional<std::reference_wrapper<const web::json::array>> expected = JsonHelper::GetRawJsonArrayFromJsonNode(jsonObject, L"Array"); + std::optional<std::reference_wrapper<const web::json::array>> expected = JSON::GetRawJsonArrayFromJsonNode(jsonObject, L"Array"); REQUIRE(expected); REQUIRE(expected.value().get().size() == 3); REQUIRE(expected.value().get().at(0).as_string() == L"ArrayValue1"); - std::optional<std::reference_wrapper<const web::json::array>> mismatchFieldTest = JsonHelper::GetRawJsonArrayFromJsonNode(jsonObject, L"Keyword"); + std::optional<std::reference_wrapper<const web::json::array>> mismatchFieldTest = JSON::GetRawJsonArrayFromJsonNode(jsonObject, L"Keyword"); REQUIRE(!mismatchFieldTest); } TEST_CASE("GetRawStringArrayFromJsonNode", "[RestSource]") { web::json::value jsonObject = GetTestJsonObject(); - std::vector<std::string> expected = JsonHelper::GetRawStringArrayFromJsonNode(jsonObject, L"Array"); + std::vector<std::string> expected = JSON::GetRawStringArrayFromJsonNode(jsonObject, L"Array"); REQUIRE(expected.size() == 3); REQUIRE(expected[0] == "ArrayValue1"); - std::vector<std::string> mismatchFieldTest = JsonHelper::GetRawStringArrayFromJsonNode(jsonObject, L"Keyword"); + std::vector<std::string> mismatchFieldTest = JSON::GetRawStringArrayFromJsonNode(jsonObject, L"Keyword"); REQUIRE(mismatchFieldTest.size() == 0); } diff --git a/src/AppInstallerCLITests/Versions.cpp b/src/AppInstallerCLITests/Versions.cpp @@ -192,6 +192,27 @@ TEST_CASE("VersionUnknownLessThanLatest", "[versions]") REQUIRE(Version::CreateUnknown() < Version::CreateLatest()); } +TEST_CASE("VersionIsEmpty", "[versions]") +{ + REQUIRE(Version{}.IsEmpty()); + REQUIRE(Version{""}.IsEmpty()); + REQUIRE(!Version{"1"}.IsEmpty()); + REQUIRE(!Version{"0"}.IsEmpty()); + + Version v{ "1" }; + REQUIRE(!v.IsEmpty()); + v.Assign(""); + REQUIRE(v.IsEmpty()); +} + +TEST_CASE("VersionPartAt", "[versions]") +{ + REQUIRE(Version{}.PartAt(0).Integer == 0); + REQUIRE(Version{"1"}.PartAt(0).Integer == 1); + REQUIRE(Version{"1"}.PartAt(1).Integer == 0); + REQUIRE(Version{"1"}.PartAt(9999).Integer == 0); +} + TEST_CASE("UInt64Version_Success_FourParts", "[versions]") { Version expectedVersion("1.2.3.4"); diff --git a/src/AppInstallerCLITests/main.cpp b/src/AppInstallerCLITests/main.cpp @@ -154,7 +154,7 @@ int main(int argc, char** argv) Runtime::TestHook_SetPathOverride(Runtime::PathName::LocalState, Runtime::GetPathTo(Runtime::PathName::LocalState) / "Tests"); Runtime::TestHook_SetPathOverride(Runtime::PathName::UserFileSettings, Runtime::GetPathTo(Runtime::PathName::UserFileSettings) / "Tests"); Runtime::TestHook_SetPathOverride(Runtime::PathName::StandardSettings, Runtime::GetPathTo(Runtime::PathName::StandardSettings) / "Tests"); - Runtime::TestHook_SetPathOverride(Runtime::PathName::SecureSettings, Runtime::GetPathTo(Runtime::PathName::Temp) / "WinGet_SecureSettings_Tests"); + Runtime::TestHook_SetPathOverride(Runtime::PathName::SecureSettings, Runtime::GetPathTo(Runtime::PathName::StandardSettings) / "WinGet_SecureSettings_Tests"); int result = Catch::Session().run(static_cast<int>(args.size()), args.data()); diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj @@ -176,10 +176,10 @@ <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'"> <ClCompile> <Optimization>Disabled</Optimization> - <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD</PreprocessorDefinitions> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <PreprocessorDefinitions>_NO_ASYNCRTIMP;_DEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD</PreprocessorDefinitions> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">true</TreatWarningAsError> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">true</TreatWarningAsError> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</TreatWarningAsError> @@ -199,8 +199,8 @@ </ItemDefinitionGroup> <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'"> <ClCompile> - <PreprocessorDefinitions>WIN32;%(PreprocessorDefinitions);CLICOREDLLBUILD</PreprocessorDefinitions> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <PreprocessorDefinitions>_NO_ASYNCRTIMP;WIN32;%(PreprocessorDefinitions);CLICOREDLLBUILD</PreprocessorDefinitions> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</TreatWarningAsError> <SDLCheck Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</SDLCheck> <EnablePREfast Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</EnablePREfast> @@ -214,11 +214,11 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD</PreprocessorDefinitions> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <PreprocessorDefinitions>_NO_ASYNCRTIMP;NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD</PreprocessorDefinitions> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">true</TreatWarningAsError> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">true</TreatWarningAsError> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</TreatWarningAsError> @@ -247,8 +247,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD;WINGET_DISABLE_FOR_FUZZING</PreprocessorDefinitions> - <AdditionalIncludeDirectories>$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <PreprocessorDefinitions>_NO_ASYNCRTIMP;NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD;WINGET_DISABLE_FOR_FUZZING</PreprocessorDefinitions> + <AdditionalIncludeDirectories>$(ProjectDir);$(ProjectDir)Public;$(ProjectDir)Telemetry;$(ProjectDir)..\binver;$(ProjectDir)..\YamlCppLib\libyaml\include;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <TreatWarningAsError>true</TreatWarningAsError> <LanguageStandard>stdcpp17</LanguageStandard> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -286,7 +286,6 @@ <ClInclude Include="HttpStream\HttpClientWrapper.h" /> <ClInclude Include="HttpStream\HttpLocalCache.h" /> <ClInclude Include="HttpStream\HttpRandomAccessStream.h" /> - <ClInclude Include="JsonUtil.h" /> <ClInclude Include="pch.h" /> <ClInclude Include="Public\AppInstallerDateTime.h" /> <ClInclude Include="Public\AppInstallerDeployment.h" /> @@ -307,6 +306,7 @@ <ClInclude Include="Public\winget\ExperimentalFeature.h" /> <ClInclude Include="Public\winget\ExtensionCatalog.h" /> <ClInclude Include="Public\winget\JsonSchemaValidation.h" /> + <ClInclude Include="Public\winget\JsonUtil.h" /> <ClInclude Include="Public\winget\Locale.h" /> <ClInclude Include="Public\winget\LocIndependent.h" /> <ClInclude Include="Public\winget\ManagedFile.h" /> diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters @@ -111,9 +111,6 @@ <ClInclude Include="Public\winget\UserSettings.h"> <Filter>Public\winget</Filter> </ClInclude> - <ClInclude Include="JsonUtil.h"> - <Filter>Header Files</Filter> - </ClInclude> <ClInclude Include="Public\winget\ExperimentalFeature.h"> <Filter>Public\winget</Filter> </ClInclude> @@ -195,6 +192,9 @@ <ClInclude Include="Public\winget\PortableARPEntry.h"> <Filter>Public\winget</Filter> </ClInclude> + <ClInclude Include="Public\winget\JsonUtil.h"> + <Filter>Public\winget</Filter> + </ClInclude> <ClInclude Include="Public\winget\ManagedFile.h"> <Filter>Public\winget</Filter> </ClInclude> diff --git a/src/AppInstallerCommonCore/Downloader.cpp b/src/AppInstallerCommonCore/Downloader.cpp @@ -168,7 +168,7 @@ namespace AppInstaller::Utility // Only Installers should be downloaded with DO currently, as: // - Index :: Constantly changing blob at same location is not what DO is for - // - Manifest :: DO overhead is not needed for small files + // - Manifest / InstallerMetadataCollectionInput :: DO overhead is not needed for small files // - WinGetUtil :: Intentionally not using DO at this time if (type == DownloadType::Installer) { diff --git a/src/AppInstallerCommonCore/JsonUtil.cpp b/src/AppInstallerCommonCore/JsonUtil.cpp @@ -1,10 +1,10 @@ // Copyright(c) Microsoft Corporation. // Licensed under the MIT License. #include "pch.h" -#include "JsonUtil.h" +#include "winget/JsonUtil.h" #include "AppInstallerStrings.h" -namespace AppInstaller::Utility +namespace AppInstaller::JSON { template<> std::optional<std::string> GetValue(const Json::Value& node) @@ -67,5 +67,158 @@ namespace AppInstaller::Utility return std::nullopt; } -} + utility::string_t GetUtilityString(std::string_view nodeName) + { + return utility::conversions::to_string_t(nodeName.data()); + } + + web::json::value GetStringValue(std::string_view value) + { + return web::json::value::string(Utility::ConvertToUTF16(value)); + } + + std::optional<std::reference_wrapper<const web::json::value>> GetJsonValueFromNode(const web::json::value& node, const utility::string_t& keyName) + { + if (node.is_null() || !node.has_field(keyName)) + { + return {}; + } + + return node.at(keyName); + } + + std::optional<std::string> GetRawStringValueFromJsonValue(const web::json::value& value) + { + if (value.is_null() || !value.is_string()) + { + return {}; + } + + return utility::conversions::to_utf8string(value.as_string()); + } + + std::optional<std::string> GetRawStringValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName) + { + std::optional<std::reference_wrapper<const web::json::value>> jsonValue = GetJsonValueFromNode(node, keyName); + + if (jsonValue) + { + return GetRawStringValueFromJsonValue(jsonValue.value().get()); + } + + return {}; + } + + std::optional<int> GetRawIntValueFromJsonValue(const web::json::value& value) + { + if (value.is_null() || !value.is_integer()) + { + return {}; + } + + return value.as_integer(); + } + + std::optional<int> GetRawIntValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName) + { + std::optional<std::reference_wrapper<const web::json::value>> jsonValue = GetJsonValueFromNode(node, keyName); + + if (jsonValue) + { + return GetRawIntValueFromJsonValue(jsonValue.value().get()); + } + + return {}; + } + + std::optional<bool> GetRawBoolValueFromJsonValue(const web::json::value& value) + { + if (value.is_null() || !value.is_boolean()) + { + return {}; + } + + return value.as_bool(); + } + + std::optional<bool> GetRawBoolValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName) + { + std::optional<std::reference_wrapper<const web::json::value>> jsonValue = GetJsonValueFromNode(node, keyName); + + if (jsonValue) + { + return GetRawBoolValueFromJsonValue(jsonValue.value().get()); + } + + return {}; + } + + std::optional<std::reference_wrapper<const web::json::array>> GetRawJsonArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName) + { + std::optional<std::reference_wrapper<const web::json::value>> jsonValue = GetJsonValueFromNode(node, keyName); + + if (!jsonValue || !jsonValue.value().get().is_array()) + { + return {}; + } + + return jsonValue.value().get().as_array(); + } + + std::vector<std::string> GetRawStringArrayFromJsonNode( + const web::json::value& node, const utility::string_t& keyName) + { + std::optional<std::reference_wrapper<const web::json::array>> arrayValue = GetRawJsonArrayFromJsonNode(node, keyName); + + std::vector<std::string> result; + if (!arrayValue) + { + return result; + } + + for (auto& value : arrayValue.value().get()) + { + std::optional<std::string> item = GetRawStringValueFromJsonValue(value); + if (item) + { + result.emplace_back(std::move(item.value())); + } + } + + return result; + } + + std::set<std::string> GetRawStringSetFromJsonNode( + const web::json::value& node, const utility::string_t& keyName) + { + std::optional<std::reference_wrapper<const web::json::array>> arrayValue = GetRawJsonArrayFromJsonNode(node, keyName); + + std::set<std::string> result; + if (!arrayValue) + { + return result; + } + + for (auto& value : arrayValue.value().get()) + { + std::optional<std::string> item = GetRawStringValueFromJsonValue(value); + if (item) + { + result.emplace(std::move(item.value())); + } + } + + return result; + } + + bool IsValidNonEmptyStringValue(std::optional<std::string>& value) + { + if (Utility::IsEmptyOrWhitespace(value.value_or(""))) + { + return false; + } + + return true; + } +} diff --git a/src/AppInstallerCommonCore/JsonUtil.h b/src/AppInstallerCommonCore/JsonUtil.h @@ -1,27 +0,0 @@ -#pragma once -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -#pragma once -#include <json.h> - -#include <optional> -#include <string> -#include <vector> - -namespace AppInstaller::Utility -{ - template<class T> - std::optional<T> GetValue(const Json::Value& node); - - template<> - std::optional<std::string> GetValue<std::string>(const Json::Value& node); - - template<> - std::optional<uint32_t> GetValue<uint32_t>(const Json::Value& node); - - template<> - std::optional<bool> GetValue<bool>(const Json::Value& node); - - template<> - std::optional<std::vector<std::string>> GetValue<std::vector<std::string>>(const Json::Value& node); -} diff --git a/src/AppInstallerCommonCore/Public/AppInstallerDownloader.h b/src/AppInstallerCommonCore/Public/AppInstallerDownloader.h @@ -22,6 +22,7 @@ namespace AppInstaller::Utility Manifest, WinGetUtil, Installer, + InstallerMetadataCollectionInput, }; // Extra metadata about a download for use by certain downloaders (Delivery Optimization for instance). diff --git a/src/AppInstallerCommonCore/Public/AppInstallerVersions.h b/src/AppInstallerCommonCore/Public/AppInstallerVersions.h @@ -50,7 +50,7 @@ namespace AppInstaller::Utility Version(Version baseVersion, ApproximateComparator approximateComparator); // Resets the version's value to the input. - virtual void Assign(std::string&& version, std::string_view splitChars = DefaultSplitChars); + virtual void Assign(std::string version, std::string_view splitChars = DefaultSplitChars); // Gets the full version string used to construct the Version. const std::string& ToString() const { return m_version; } @@ -74,9 +74,15 @@ namespace AppInstaller::Utility // Returns a Version that will return true for IsUnknown static Version CreateUnknown(); + // Gets a bool indicating whether the full version string is empty. + // Does not indicate that Parts is empty; for instance when "0.0" is given, + // this will be false while GetParts().empty() would be true. + bool IsEmpty() const { return m_version.empty(); } + // An individual version part in between split characters. struct Part { + Part() = default; Part(uint64_t integer) : Integer(integer) {} Part(const std::string& part); Part(uint64_t integer, std::string other); @@ -92,6 +98,9 @@ namespace AppInstaller::Utility // Gets the part breakdown for a given version; used for tests. const std::vector<Part>& GetParts() const { return m_parts; } + // Gets the part at the given index; or the implied zero part if past the end. + const Part& PartAt(size_t index) const; + // Returns if the version is an approximate version. bool IsApproximate() const { return m_approximateComparator != ApproximateComparator::None; } @@ -119,7 +128,7 @@ namespace AppInstaller::Utility UInt64Version(const std::string& version, std::string_view splitChars = DefaultSplitChars) : UInt64Version(std::string(version), splitChars) {} - void Assign(std::string&& version, std::string_view splitChars = DefaultSplitChars) override; + void Assign(std::string version, std::string_view splitChars = DefaultSplitChars) override; void Assign(UINT64 version); UINT64 Major() const { return m_parts.size() > 0 ? m_parts[0].Integer : 0; } diff --git a/src/AppInstallerCommonCore/Public/winget/JsonSchemaValidation.h b/src/AppInstallerCommonCore/Public/winget/JsonSchemaValidation.h @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. #pragma once -#include <json.h> +#include <json/json.h> namespace AppInstaller::JsonSchema { diff --git a/src/AppInstallerCommonCore/Public/winget/JsonUtil.h b/src/AppInstallerCommonCore/Public/winget/JsonUtil.h @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include <json/json.h> + +// Disable dllimport for cpprest JSON in any downstream consumers +#ifndef _NO_ASYNCRTIMP +#define _NO_ASYNCRTIMP +#endif +#include <cpprest/json.h> + +#include <optional> +#include <string> +#include <vector> + +namespace AppInstaller::JSON +{ + // For JSON CPP Lib + template<class T> + std::optional<T> GetValue(const Json::Value& node); + + template<> + std::optional<std::string> GetValue<std::string>(const Json::Value& node); + + template<> + std::optional<uint32_t> GetValue<uint32_t>(const Json::Value& node); + + template<> + std::optional<bool> GetValue<bool>(const Json::Value& node); + + template<> + std::optional<std::vector<std::string>> GetValue<std::vector<std::string>>(const Json::Value& node); + + // For cpprestsdk JSON + std::optional<std::reference_wrapper<const web::json::value>> GetJsonValueFromNode(const web::json::value& node, const utility::string_t& keyName); + + std::optional<std::string> GetRawStringValueFromJsonValue(const web::json::value& value); + + std::optional<std::string> GetRawStringValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName); + + std::optional<bool> GetRawBoolValueFromJsonValue(const web::json::value& value); + + std::optional<bool> GetRawBoolValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName); + + std::optional<std::reference_wrapper<const web::json::array>> GetRawJsonArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName); + + std::vector<std::string> GetRawStringArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName); + std::set<std::string> GetRawStringSetFromJsonNode(const web::json::value& node, const utility::string_t& keyName); + + std::optional<int> GetRawIntValueFromJsonValue(const web::json::value& value); + + std::optional<int> GetRawIntValueFromJsonNode(const web::json::value& value, const utility::string_t& keyName); + + utility::string_t GetUtilityString(std::string_view nodeName); + + web::json::value GetStringValue(std::string_view value); + + bool IsValidNonEmptyStringValue(std::optional<std::string>& value); +} diff --git a/src/AppInstallerCommonCore/Public/winget/ManifestSchemaValidation.h b/src/AppInstallerCommonCore/Public/winget/ManifestSchemaValidation.h @@ -4,7 +4,7 @@ #include "ManifestCommon.h" #include "ManifestValidation.h" -#include <json.h> +#include <json/json.h> namespace AppInstaller::Manifest::YamlParser { diff --git a/src/AppInstallerCommonCore/ThreadGlobals.cpp b/src/AppInstallerCommonCore/ThreadGlobals.cpp @@ -1,11 +1,11 @@ -#include "pch.h" -#include "Public/winget/ThreadGlobals.h" - -namespace AppInstaller::ThreadLocalStorage -{ - using namespace AppInstaller::Logging; - - // Set and return Globals for Current Thread +#include "pch.h" +#include "Public/winget/ThreadGlobals.h" + +namespace AppInstaller::ThreadLocalStorage +{ + using namespace AppInstaller::Logging; + + // Set and return Globals for Current Thread static ThreadGlobals* SetOrGetThreadGlobals(bool setThreadGlobals, ThreadGlobals* pThreadGlobals = nullptr); ThreadGlobals::ThreadGlobals(ThreadGlobals& parent, create_sub_thread_globals_t) @@ -17,67 +17,67 @@ namespace AppInstaller::ThreadLocalStorage std::call_once(m_loggerInitOnceFlag, []() {}); } - DiagnosticLogger& ThreadGlobals::GetDiagnosticLogger() - { - return *(m_pDiagnosticLogger); + DiagnosticLogger& ThreadGlobals::GetDiagnosticLogger() + { + return *(m_pDiagnosticLogger); } - TelemetryTraceLogger& ThreadGlobals::GetTelemetryLogger() - { - return *(m_pTelemetryLogger); - } - - std::unique_ptr<PreviousThreadGlobals> ThreadGlobals::SetForCurrentThread() - { - Initialize(); - - std::unique_ptr<PreviousThreadGlobals> p_prevThreadGlobals = std::make_unique<PreviousThreadGlobals>(SetOrGetThreadGlobals(true, this)); - - return p_prevThreadGlobals; - } - - void ThreadGlobals::Initialize() - { - try - { - std::call_once(m_loggerInitOnceFlag, [this]() - { - m_pDiagnosticLogger = std::make_unique<DiagnosticLogger>(); - m_pTelemetryLogger = std::make_unique<TelemetryTraceLogger>(); - - // The above make_unique for TelemetryTraceLogger will either create an object or will throw which is caught below. - m_pTelemetryLogger->Initialize(); - }); - } - catch (...) - { - // May throw std::system_error if any condition prevents calls to call_once from executing as specified - // May throw std::bad_alloc or any exception thrown by the constructor of TelemetryTraceLogger - // Loggers are best effort and shouldn't block core functionality. So eat up the exceptions here - } - } - - ThreadGlobals* ThreadGlobals::GetForCurrentThread() - { - return SetOrGetThreadGlobals(false); - } - - ThreadGlobals* SetOrGetThreadGlobals(bool setThreadGlobals, ThreadGlobals* pThreadGlobals) - { - thread_local AppInstaller::ThreadLocalStorage::ThreadGlobals* t_pThreadGlobals = nullptr; - - if (setThreadGlobals == true) - { - AppInstaller::ThreadLocalStorage::ThreadGlobals* previous_pThreadGlobals = t_pThreadGlobals; - t_pThreadGlobals = pThreadGlobals; - return previous_pThreadGlobals; - } - - return t_pThreadGlobals; - } - - PreviousThreadGlobals::~PreviousThreadGlobals() - { - std::ignore = SetOrGetThreadGlobals(true, m_previous); - } -} + TelemetryTraceLogger& ThreadGlobals::GetTelemetryLogger() + { + return *(m_pTelemetryLogger); + } + + std::unique_ptr<PreviousThreadGlobals> ThreadGlobals::SetForCurrentThread() + { + Initialize(); + + std::unique_ptr<PreviousThreadGlobals> p_prevThreadGlobals = std::make_unique<PreviousThreadGlobals>(SetOrGetThreadGlobals(true, this)); + + return p_prevThreadGlobals; + } + + void ThreadGlobals::Initialize() + { + try + { + std::call_once(m_loggerInitOnceFlag, [this]() + { + m_pDiagnosticLogger = std::make_unique<DiagnosticLogger>(); + m_pTelemetryLogger = std::make_unique<TelemetryTraceLogger>(); + + // The above make_unique for TelemetryTraceLogger will either create an object or will throw which is caught below. + m_pTelemetryLogger->Initialize(); + }); + } + catch (...) + { + // May throw std::system_error if any condition prevents calls to call_once from executing as specified + // May throw std::bad_alloc or any exception thrown by the constructor of TelemetryTraceLogger + // Loggers are best effort and shouldn't block core functionality. So eat up the exceptions here + } + } + + ThreadGlobals* ThreadGlobals::GetForCurrentThread() + { + return SetOrGetThreadGlobals(false); + } + + ThreadGlobals* SetOrGetThreadGlobals(bool setThreadGlobals, ThreadGlobals* pThreadGlobals) + { + thread_local AppInstaller::ThreadLocalStorage::ThreadGlobals* t_pThreadGlobals = nullptr; + + if (setThreadGlobals == true) + { + AppInstaller::ThreadLocalStorage::ThreadGlobals* previous_pThreadGlobals = t_pThreadGlobals; + t_pThreadGlobals = pThreadGlobals; + return previous_pThreadGlobals; + } + + return t_pThreadGlobals; + } + + PreviousThreadGlobals::~PreviousThreadGlobals() + { + std::ignore = SetOrGetThreadGlobals(true, m_previous); + } +} diff --git a/src/AppInstallerCommonCore/UserSettings.cpp b/src/AppInstallerCommonCore/UserSettings.cpp @@ -4,7 +4,7 @@ #include "AppInstallerRuntime.h" #include "AppInstallerLanguageUtilities.h" #include "AppInstallerLogging.h" -#include "JsonUtil.h" +#include "winget/JsonUtil.h" #include "winget/Settings.h" #include "winget/UserSettings.h" @@ -17,6 +17,7 @@ namespace AppInstaller::Settings using namespace Runtime; using namespace Utility; using namespace Logging; + using namespace JSON; static constexpr std::string_view s_SettingEmpty = R"({ diff --git a/src/AppInstallerCommonCore/Versions.cpp b/src/AppInstallerCommonCore/Versions.cpp @@ -39,7 +39,7 @@ namespace AppInstaller::Utility } } - void Version::Assign(std::string&& version, std::string_view splitChars) + void Version::Assign(std::string version, std::string_view splitChars) { m_version = std::move(version); @@ -232,6 +232,20 @@ namespace AppInstaller::Utility return result; } + const Version::Part& Version::PartAt(size_t index) const + { + static Part s_zero{}; + + if (index < m_parts.size()) + { + return m_parts[index]; + } + else + { + return s_zero; + } + } + bool Version::IsBaseVersionLatest() const { return (m_parts.size() == 1 && m_parts[0].Integer == 0 && Utility::CaseInsensitiveEquals(m_parts[0].Other, s_Version_Part_Latest)); @@ -394,7 +408,7 @@ namespace AppInstaller::Utility Assign(std::move(version), splitChars); } - void UInt64Version::Assign(std::string&& version, std::string_view splitChars) + void UInt64Version::Assign(std::string version, std::string_view splitChars) { Version::Assign(std::move(version), splitChars); diff --git a/src/AppInstallerCommonCore/pch.h b/src/AppInstallerCommonCore/pch.h @@ -21,7 +21,9 @@ #define YAML_DECLARE_STATIC #include <yaml.h> -#include <json.h> +// TODO: See if we can get down to having just one JSON parser... +#include <json/json.h> +#include <cpprest/json.h> #pragma warning( push ) #pragma warning ( disable : 4458 4100 4702 6031 ) diff --git a/src/AppInstallerRepositoryCore/ARPCorrelation.cpp b/src/AppInstallerRepositoryCore/ARPCorrelation.cpp @@ -56,17 +56,85 @@ namespace AppInstaller::Repository::Correlation } #endif - ARPCorrelationResult FindARPEntryForNewlyInstalledPackage( + // Find the best match using heuristics + std::shared_ptr<IPackageVersion> FindARPEntryForNewlyInstalledPackageWithHeuristics( const Manifest::Manifest& manifest, - const std::vector<ARPEntrySnapshot>& arpSnapshot, - Source& arpSource) + const std::vector<ARPEntry>& arpEntries) { - AICLI_LOG(Repo, Verbose, << "Finding ARP entry matching newly installed package"); + // TODO: In the future we can make different passes with different algorithms until we find a match + return FindARPEntryForNewlyInstalledPackageWithHeuristics(manifest, arpEntries, IARPMatchConfidenceAlgorithm::Instance()); + } + + std::shared_ptr<IPackageVersion> FindARPEntryForNewlyInstalledPackageWithHeuristics( + const AppInstaller::Manifest::Manifest& manifest, + const std::vector<ARPEntry>& arpEntries, + IARPMatchConfidenceAlgorithm& algorithm) + { + AICLI_LOG(Repo, Verbose, << "Looking for best match in ARP for manifest " << manifest.Id); + + algorithm.Init(manifest); + + std::optional<ARPEntry> bestMatch; + double bestScore = 0; + + for (const auto& arpEntry : arpEntries) + { + auto score = algorithm.ComputeConfidence(arpEntry); + AICLI_LOG(Repo, Verbose, << "Match confidence for " << arpEntry.Entry->GetProperty(PackageProperty::Id) << ": " << score); + + if (score < MatchingThreshold) + { + AICLI_LOG(Repo, Verbose, << "Score is lower than threshold"); + continue; + } + + if (!bestMatch || bestScore < score) + { + bestMatch = arpEntry; + bestScore = score; + } + } - std::vector<Correlation::ARPEntry> changedArpEntries; - std::vector<Correlation::ARPEntry> existingArpEntries; + if (bestMatch) + { + AICLI_LOG(Repo, Verbose, << "Best match is " << bestMatch->Entry->GetProperty(PackageProperty::Id)); + } + else + { + AICLI_LOG(Repo, Verbose, << "No ARP entry had a correlation score surpassing the required threshold"); + } - for (auto& entry : arpSource.Search({}).Matches) + return bestMatch ? bestMatch->Entry->GetInstalledVersion() : nullptr; + } + + void ARPCorrelationData::CapturePreInstallSnapshot() + { + ProgressCallback empty; + Repository::Source preInstallARP = Repository::Source(PredefinedSource::ARP); + preInstallARP.Open(empty); + + for (const auto& entry : preInstallARP.Search({}).Matches) + { + auto installed = entry.Package->GetInstalledVersion(); + if (installed) + { + m_preInstallSnapshot.emplace_back(std::make_tuple( + entry.Package->GetProperty(PackageProperty::Id), + installed->GetProperty(PackageVersionProperty::Version), + installed->GetProperty(PackageVersionProperty::Channel))); + } + } + + std::sort(m_preInstallSnapshot.begin(), m_preInstallSnapshot.end()); + } + + void ARPCorrelationData::CapturePostInstallSnapshot() + { + ProgressCallback empty; + m_postInstallSnapshotSource = Repository::Source(PredefinedSource::ARP); + m_postInstallSnapshotSource.Open(empty); + + for (auto& entry : m_postInstallSnapshotSource.Search({}).Matches) { auto installed = entry.Package->GetInstalledVersion(); @@ -77,17 +145,22 @@ namespace AppInstaller::Repository::Correlation installed->GetProperty(PackageVersionProperty::Version), installed->GetProperty(PackageVersionProperty::Channel)); - auto itr = std::lower_bound(arpSnapshot.begin(), arpSnapshot.end(), entryKey); - if (itr == arpSnapshot.end() || *itr != entryKey) + auto itr = std::lower_bound(m_preInstallSnapshot.begin(), m_preInstallSnapshot.end(), entryKey); + if (itr == m_preInstallSnapshot.end() || *itr != entryKey) { - changedArpEntries.emplace_back(entry.Package, true); + m_postInstallSnapshot.emplace_back(entry.Package, true); } else { - existingArpEntries.emplace_back(entry.Package, false); + m_postInstallSnapshot.emplace_back(entry.Package, false); } } } + } + + ARPCorrelationResult ARPCorrelationData::CorrelateForNewlyInstalled(const Manifest::Manifest& manifest) + { + AICLI_LOG(Repo, Verbose, << "Finding ARP entry matching newly installed package"); // Also attempt to find the entry based on the manifest data @@ -143,20 +216,23 @@ namespace AppInstaller::Repository::Correlation // Don't execute this search if it would just find everything if (!manifestSearchRequest.IsForEverything()) { - findByManifest = arpSource.Search(manifestSearchRequest); + findByManifest = m_postInstallSnapshotSource.Search(manifestSearchRequest); } // Cross reference the changes with the search results std::vector<std::shared_ptr<IPackage>> packagesInBoth; - for (const auto& change : changedArpEntries) + for (const auto& change : m_postInstallSnapshot) { - for (const auto& byManifest : findByManifest.Matches) + if (change.IsNewOrUpdated) { - if (change.Entry->IsSame(byManifest.Package.get())) + for (const auto& byManifest : findByManifest.Matches) { - packagesInBoth.emplace_back(change.Entry); - break; + if (change.Entry->IsSame(byManifest.Package.get())) + { + packagesInBoth.emplace_back(change.Entry); + break; + } } } } @@ -172,7 +248,7 @@ namespace AppInstaller::Repository::Correlation // Find the package that we are going to log ARPCorrelationResult result; // TODO: Find a good way to consider the other heuristics in these stats. - result.ChangesToARP = changedArpEntries.size(); + result.ChangesToARP = std::count_if(m_postInstallSnapshot.begin(), m_postInstallSnapshot.end(), [](const ARPEntry& e) { return e.IsNewOrUpdated; }); result.MatchesInARP = findByManifest.Matches.size(); result.CountOfIntersectionOfChangesAndMatches = packagesInBoth.size(); @@ -192,70 +268,9 @@ namespace AppInstaller::Repository::Correlation // to try and match the package with some ARP entry by assigning them scores. AICLI_LOG(Repo, Verbose, << "No exact ARP match found. Trying to find one with heuristics"); - std::vector<ARPEntry> arpEntries; - for (auto&& entry : changedArpEntries) - { - arpEntries.push_back(std::move(entry)); - } - for (auto&& entry : existingArpEntries) - { - arpEntries.push_back(std::move(entry)); - } - - result.Package = FindARPEntryForNewlyInstalledPackageWithHeuristics(manifest, arpEntries); + result.Package = FindARPEntryForNewlyInstalledPackageWithHeuristics(manifest, m_postInstallSnapshot); } return result; } - - // Find the best match using heuristics - std::shared_ptr<IPackageVersion> FindARPEntryForNewlyInstalledPackageWithHeuristics( - const Manifest::Manifest& manifest, - const std::vector<ARPEntry>& arpEntries) - { - // TODO: In the future we can make different passes with different algorithms until we find a match - return FindARPEntryForNewlyInstalledPackageWithHeuristics(manifest, arpEntries, IARPMatchConfidenceAlgorithm::Instance()); - } - - std::shared_ptr<IPackageVersion> FindARPEntryForNewlyInstalledPackageWithHeuristics( - const AppInstaller::Manifest::Manifest& manifest, - const std::vector<ARPEntry>& arpEntries, - IARPMatchConfidenceAlgorithm& algorithm) - { - AICLI_LOG(Repo, Verbose, << "Looking for best match in ARP for manifest " << manifest.Id); - - algorithm.Init(manifest); - - std::optional<ARPEntry> bestMatch; - double bestScore = 0; - - for (const auto& arpEntry : arpEntries) - { - auto score = algorithm.ComputeConfidence(arpEntry); - AICLI_LOG(Repo, Verbose, << "Match confidence for " << arpEntry.Entry->GetProperty(PackageProperty::Id) << ": " << score); - - if (score < MatchingThreshold) - { - AICLI_LOG(Repo, Verbose, << "Score is lower than threshold"); - continue; - } - - if (!bestMatch || bestScore < score) - { - bestMatch = arpEntry; - bestScore = score; - } - } - - if (bestMatch) - { - AICLI_LOG(Repo, Verbose, << "Best match is " << bestMatch->Entry->GetProperty(PackageProperty::Id)); - } - else - { - AICLI_LOG(Repo, Verbose, << "No ARP entry had a correlation score surpassing the required threshold"); - } - - return bestMatch ? bestMatch->Entry->GetInstalledVersion() : nullptr; - } -}- \ No newline at end of file +} diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -159,9 +159,9 @@ <ClCompile> <Optimization>Disabled</Optimization> <PreprocessorDefinitions>_NO_ASYNCRTIMP;_DEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD</PreprocessorDefinitions> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">true</TreatWarningAsError> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">true</TreatWarningAsError> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</TreatWarningAsError> @@ -182,7 +182,7 @@ <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'"> <ClCompile> <PreprocessorDefinitions>_NO_ASYNCRTIMP;WIN32;%(PreprocessorDefinitions);CLICOREDLLBUILD</PreprocessorDefinitions> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</TreatWarningAsError> <SDLCheck Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</SDLCheck> <EnablePREfast Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</EnablePREfast> @@ -197,10 +197,10 @@ <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>_NO_ASYNCRTIMP;NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD</PreprocessorDefinitions> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(ProjectDir);$(ProjectDir)\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">true</TreatWarningAsError> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">true</TreatWarningAsError> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</TreatWarningAsError> @@ -277,6 +277,8 @@ <ClInclude Include="PackageTrackingCatalogSourceFactory.h" /> <ClInclude Include="pch.h" /> <ClInclude Include="Public\winget\ARPCorrelation.h" /> + <ClInclude Include="Public\winget\InstallerMetadataCollectionContext.h" /> + <ClInclude Include="Public\winget\ManifestJSONParser.h" /> <ClInclude Include="Public\winget\ARPCorrelationAlgorithms.h" /> <ClInclude Include="Public\winget\PackageTrackingCatalog.h" /> <ClInclude Include="Public\winget\RepositorySearch.h" /> @@ -295,7 +297,6 @@ <ClInclude Include="Rest\Schema\HttpClientHelper.h" /> <ClInclude Include="Rest\Schema\InformationResponseDeserializer.h" /> <ClInclude Include="Rest\Schema\IRestClient.h" /> - <ClInclude Include="Rest\Schema\JsonHelper.h" /> <ClInclude Include="Rest\Schema\RestHelper.h" /> <ClInclude Include="SourceFactory.h" /> <ClInclude Include="SourceList.h" /> @@ -319,6 +320,8 @@ <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">NotUsing</PrecompiledHeader> <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">NotUsing</PrecompiledHeader> </ClCompile> + <ClCompile Include="InstallerMetadataCollectionContext.cpp" /> + <ClCompile Include="ManifestJSONParser.cpp" /> <ClCompile Include="Microsoft\ARPHelper.cpp" /> <ClCompile Include="Microsoft\ConfigurableTestSourceFactory.cpp" /> <ClCompile Include="Microsoft\PredefinedInstalledSourceFactory.cpp" /> @@ -362,7 +365,6 @@ <ClCompile Include="Rest\Schema\1_1\RestInterface_1_1.cpp" /> <ClCompile Include="Rest\Schema\HttpClientHelper.cpp" /> <ClCompile Include="Rest\Schema\InformationResponseDeserializer.cpp" /> - <ClCompile Include="Rest\Schema\JsonHelper.cpp" /> <ClCompile Include="Rest\Schema\RestHelper.cpp" /> <ClCompile Include="SourceList.cpp" /> <ClCompile Include="SourcePolicy.cpp" /> diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -204,9 +204,6 @@ <ClInclude Include="Rest\Schema\1_0\Json\SearchResponseDeserializer.h"> <Filter>Rest\Schema\1_0\Json</Filter> </ClInclude> - <ClInclude Include="Rest\Schema\JsonHelper.h"> - <Filter>Rest\Schema</Filter> - </ClInclude> <ClInclude Include="Rest\Schema\CommonRestConstants.h"> <Filter>Rest\Schema</Filter> </ClInclude> @@ -270,8 +267,14 @@ <ClInclude Include="Public\winget\ARPCorrelation.h"> <Filter>Public\winget</Filter> </ClInclude> + <ClInclude Include="Public\winget\ManifestJSONParser.h"> + <Filter>Public\winget</Filter> + </ClInclude> + <ClInclude Include="Public\winget\InstallerMetadataCollectionContext.h"> + <Filter>Public\winget</Filter> + </ClInclude> <ClInclude Include="Public\winget\ARPCorrelationAlgorithms.h"> - <Filter>Header Files</Filter> + <Filter>Public\winget</Filter> </ClInclude> <ClInclude Include="Microsoft\Schema\1_5\ArpVersionVirtualTable.h"> <Filter>Microsoft\Schema\1_5</Filter> @@ -386,9 +389,6 @@ <ClCompile Include="Rest\Schema\1_0\Json\SearchResponseDeserializer_1_0.cpp"> <Filter>Rest\Schema\1_0\Json</Filter> </ClCompile> - <ClCompile Include="Rest\Schema\JsonHelper.cpp"> - <Filter>Rest\Schema</Filter> - </ClCompile> <ClCompile Include="Microsoft\Schema\1_3\Interface_1_3.cpp"> <Filter>Microsoft\Schema\1_3</Filter> </ClCompile> @@ -437,6 +437,12 @@ <ClCompile Include="ARPCorrelation.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="ManifestJSONParser.cpp"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="InstallerMetadataCollectionContext.cpp"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="ARPCorrelationAlgorithms.cpp"> <Filter>Source Files</Filter> </ClCompile> diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -0,0 +1,1011 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "winget/InstallerMetadataCollectionContext.h" + +#include <AppInstallerDownloader.h> +#include <AppInstallerErrors.h> +#include <AppInstallerLogging.h> +#include <AppInstallerStrings.h> + +#include <winget/ManifestJSONParser.h> + +using namespace AppInstaller::Utility; + +namespace AppInstaller::Repository::Metadata +{ + namespace + { + struct ProductMetadataFields_1_0 + { + utility::string_t ProductVersionMin = L"productVersionMin"; + utility::string_t ProductVersionMax = L"productVersionMax"; + utility::string_t Metadata = L"metadata"; + utility::string_t InstallerHash = L"installerHash"; + utility::string_t SubmissionIdentifier = L"submissionIdentifier"; + utility::string_t Version = L"version"; + utility::string_t AppsAndFeaturesEntries = L"AppsAndFeaturesEntries"; + utility::string_t Historical = L"historical"; + + utility::string_t DisplayName = L"DisplayName"; + utility::string_t Publisher = L"Publisher"; + utility::string_t DisplayVersion = L"DisplayVersion"; + utility::string_t ProductCode = L"ProductCode"; + utility::string_t UpgradeCode = L"UpgradeCode"; + utility::string_t InstallerType = L"InstallerType"; + + utility::string_t VersionMin = L"versionMin"; + utility::string_t VersionMax = L"versionMax"; + utility::string_t Names = L"names"; + utility::string_t Publishers = L"publishers"; + utility::string_t ProductCodes = L"productCodes"; + utility::string_t UpgradeCodes = L"upgradeCodes"; + }; + + struct OutputFields_1_0 + { + utility::string_t Version = L"version"; + utility::string_t SubmissionData = L"submissionData"; + utility::string_t InstallerHash = L"installerHash"; + utility::string_t Status = L"status"; + utility::string_t Metadata = L"metadata"; + utility::string_t Diagnostics = L"diagnostics"; + + utility::string_t ErrorHR = L"errorHR"; + utility::string_t ErrorText = L"errorText"; + }; + + std::string GetRequiredString(const web::json::value& value, const utility::string_t& field) + { + auto optString = AppInstaller::JSON::GetRawStringValueFromJsonNode(value, field); + if (!optString) + { + AICLI_LOG(Repo, Error, << "Required field '" << Utility::ConvertToUTF8(field) << "' was not present"); + THROW_HR(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE); + } + return std::move(optString).value(); + } + + void AddFieldIfNotEmpty(web::json::value& value, const utility::string_t& field, std::string_view string) + { + if (!string.empty()) + { + value[field] = AppInstaller::JSON::GetStringValue(string); + } + } + + web::json::value CreateStringArray(const std::set<std::string>& values) + { + web::json::value result; + size_t index = 0; + + for (const std::string& value : values) + { + result[index++] = AppInstaller::JSON::GetStringValue(value); + } + + return result; + } + + bool AddIfNotPresentAndNotEmpty(std::set<std::string>& strings, const std::set<std::string>& filter, const std::string& string) + { + if (string.empty() || filter.find(string) != filter.end()) + { + return false; + } + + strings.emplace(string); + return true; + } + + bool AddIfNotPresentAndNotEmpty(std::set<std::string>& strings, const std::string& string) + { + return AddIfNotPresentAndNotEmpty(strings, strings, string); + } + + void AddIfNotPresent(std::set<std::string>& strings, std::set<std::string>& filter, const std::set<std::string>& inputs) + { + for (const std::string& input : inputs) + { + if (AddIfNotPresentAndNotEmpty(strings, filter, input)) + { + filter.emplace(input); + } + } + } + + void FilterAndAddToEntries(Manifest::AppsAndFeaturesEntry&& newEntry, std::vector<Manifest::AppsAndFeaturesEntry>& entries) + { + // Erase all duplicated data from the new entry + for (const auto& entry : entries) + { +#define WINGET_ERASE_IF_SAME(_value_) if (entry._value_ == newEntry._value_) { newEntry._value_.clear(); } + WINGET_ERASE_IF_SAME(DisplayName); + WINGET_ERASE_IF_SAME(DisplayVersion); + WINGET_ERASE_IF_SAME(ProductCode); + WINGET_ERASE_IF_SAME(Publisher); + WINGET_ERASE_IF_SAME(UpgradeCode); +#undef WINGET_ERASE_IF_SAME + + if (entry.InstallerType == newEntry.InstallerType) + { + newEntry.InstallerType = Manifest::InstallerTypeEnum::Unknown; + } + } + + // If anything remains, add it + if (!newEntry.DisplayName.empty() || !newEntry.DisplayVersion.empty() || !newEntry.ProductCode.empty() || + !newEntry.Publisher.empty() || !newEntry.UpgradeCode.empty() || newEntry.InstallerType != Manifest::InstallerTypeEnum::Unknown) + { + entries.emplace_back(std::move(newEntry)); + } + } + } + + void ProductMetadata::Clear() + { + SchemaVersion = {}; + ProductVersionMin = {}; + ProductVersionMax = {}; + InstallerMetadataMap.clear(); + HistoricalMetadataList.clear(); + } + + void ProductMetadata::FromJson(const web::json::value& json) + { + Clear(); + + utility::string_t versionFieldName = L"version"; + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, json.is_null()); + + SchemaVersion = Version{ GetRequiredString(json, versionFieldName) }; + AICLI_LOG(Repo, Info, << "Parsing metadata JSON version " << SchemaVersion.ToString()); + + if (SchemaVersion.PartAt(0).Integer == 1) + { + // We only have one version currently, so use that as long as the major version is 1 + FromJson_1_0(json); + } + else + { + AICLI_LOG(Repo, Error, << "Don't know how to handle metadata version " << SchemaVersion.ToString()); + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } + + // Sort the historical data with oldest last (thus b < a) + std::sort(HistoricalMetadataList.begin(), HistoricalMetadataList.end(), + [](const HistoricalMetadata& a, const HistoricalMetadata& b) { + return b.ProductVersionMin < a.ProductVersionMin; + }); + } + + web::json::value ProductMetadata::ToJson(const Utility::Version& schemaVersion, size_t maximumSizeInBytes) + { + AICLI_LOG(Repo, Info, << "Creating metadata JSON version " << schemaVersion.ToString()); + + using ToJsonFunctionPointer = web::json::value(ProductMetadata::*)(); + ToJsonFunctionPointer toJsonFunction = nullptr; + + if (schemaVersion.PartAt(0).Integer == 1) + { + // We only have one version currently, so use that as long as the major version is 1 + toJsonFunction = &ProductMetadata::ToJson_1_0; + } + else + { + AICLI_LOG(Repo, Error, << "Don't know how to handle metadata version " << schemaVersion.ToString()); + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } + + // Constrain the result based on maximum size given + web::json::value result = (this->*toJsonFunction)(); + + while (maximumSizeInBytes) + { + // Determine current size + std::ostringstream temp; + result.serialize(temp); + + std::string tempStr = temp.str(); + if (tempStr.length() > maximumSizeInBytes) + { + if (!DropOldestHistoricalData()) + { + AICLI_LOG(Repo, Error, << "Could not remove any more historical data to get under " << maximumSizeInBytes << " bytes"); + AICLI_LOG(Repo, Info, << " Smallest size was " << tempStr.length() << " bytes with value:\n" << tempStr); + THROW_HR(HRESULT_FROM_WIN32(ERROR_FILE_TOO_LARGE)); + } + result = (this->*toJsonFunction)(); + } + else + { + break; + } + } + + return result; + } + + void ProductMetadata::CopyFrom(const ProductMetadata& source, std::string_view submissionIdentifier) + { + // If the source has no installer metadata, consider it empty + if (source.InstallerMetadataMap.empty()) + { + return; + } + + // With the same submission, just copy over all of the data + if (source.InstallerMetadataMap.begin()->second.SubmissionIdentifier == submissionIdentifier) + { + *this = source; + return; + } + + // This is a new submission, so we must move all of the data to historical and update the older historical data + // First, create a new historical entry for the current metadata + HistoricalMetadata currentHistory; + + currentHistory.ProductVersionMin = source.ProductVersionMin; + currentHistory.ProductVersionMax = source.ProductVersionMax; + + for (const auto& metadataItem : source.InstallerMetadataMap) + { + for (const auto& entry : metadataItem.second.AppsAndFeaturesEntries) + { + AddIfNotPresentAndNotEmpty(currentHistory.Names, entry.DisplayName); + AddIfNotPresentAndNotEmpty(currentHistory.Publishers, entry.Publisher); + AddIfNotPresentAndNotEmpty(currentHistory.ProductCodes, entry.ProductCode); + AddIfNotPresentAndNotEmpty(currentHistory.UpgradeCodes, entry.UpgradeCode); + } + } + + // Copy the data in so that we can continue using currentHistory to track all strings + HistoricalMetadataList.emplace_back(currentHistory); + + // Now, copy over the other historical data, filtering out anything we have seen + for (const auto& historical : source.HistoricalMetadataList) + { + HistoricalMetadata copied; + copied.ProductVersionMin = historical.ProductVersionMin; + copied.ProductVersionMax = historical.ProductVersionMax; + AddIfNotPresent(copied.Names, currentHistory.Names, historical.Names); + AddIfNotPresent(copied.Publishers, currentHistory.Publishers, historical.Publishers); + AddIfNotPresent(copied.ProductCodes, currentHistory.ProductCodes, historical.ProductCodes); + AddIfNotPresent(copied.UpgradeCodes, currentHistory.UpgradeCodes, historical.UpgradeCodes); + + if (!copied.Names.empty() || !copied.Publishers.empty() || !copied.ProductCodes.empty() || !copied.UpgradeCodes.empty()) + { + HistoricalMetadataList.emplace_back(std::move(copied)); + } + } + } + + void ProductMetadata::FromJson_1_0(const web::json::value& json) + { + AICLI_LOG(Repo, Info, << "Parsing metadata JSON 1.0 fields"); + + ProductMetadataFields_1_0 fields; + + auto productVersionMinString = AppInstaller::JSON::GetRawStringValueFromJsonNode(json, fields.ProductVersionMin); + if (productVersionMinString) + { + ProductVersionMin = Version{ std::move(productVersionMinString).value() }; + } + + auto productVersionMaxString = AppInstaller::JSON::GetRawStringValueFromJsonNode(json, fields.ProductVersionMax); + if (productVersionMaxString) + { + ProductVersionMax = Version{ std::move(productVersionMaxString).value() }; + } + + // The 1.0 version of metadata uses the 1.1 version of REST + JSON::ManifestJSONParser parser{ Version{ "1.1" } }; + + std::string submissionIdentifierVerification; + + auto metadataArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(json, fields.Metadata); + if (metadataArray) + { + for (const auto& item : metadataArray->get()) + { + std::string installerHashString = GetRequiredString(item, fields.InstallerHash); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, InstallerMetadataMap.find(installerHashString) != InstallerMetadataMap.end()); + + InstallerMetadata installerMetadata; + + installerMetadata.SubmissionIdentifier = GetRequiredString(item, fields.SubmissionIdentifier); + if (submissionIdentifierVerification.empty()) + { + submissionIdentifierVerification = installerMetadata.SubmissionIdentifier; + } + else if (submissionIdentifierVerification != installerMetadata.SubmissionIdentifier) + { + AICLI_LOG(Repo, Error, << "Different submission identifiers found in metadata: '" << + submissionIdentifierVerification << "' and '" << installerMetadata.SubmissionIdentifier << "'"); + THROW_HR(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE); + } + + auto appsAndFeatures = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(item, fields.AppsAndFeaturesEntries); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !appsAndFeatures); + installerMetadata.AppsAndFeaturesEntries = parser.DeserializeAppsAndFeaturesEntries(appsAndFeatures.value()); + + InstallerMetadataMap[installerHashString] = std::move(installerMetadata); + } + } + + auto historicalArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(json, fields.Historical); + if (historicalArray) + { + for (const auto& item : historicalArray->get()) + { + HistoricalMetadata historicalMetadata; + + historicalMetadata.ProductVersionMin = Version{ GetRequiredString(item, fields.VersionMin) }; + historicalMetadata.ProductVersionMax = Version{ GetRequiredString(item, fields.VersionMax) }; + historicalMetadata.Names = AppInstaller::JSON::GetRawStringSetFromJsonNode(item, fields.Names); + historicalMetadata.Publishers = AppInstaller::JSON::GetRawStringSetFromJsonNode(item, fields.Publishers); + historicalMetadata.ProductCodes = AppInstaller::JSON::GetRawStringSetFromJsonNode(item, fields.ProductCodes); + historicalMetadata.UpgradeCodes = AppInstaller::JSON::GetRawStringSetFromJsonNode(item, fields.UpgradeCodes); + + HistoricalMetadataList.emplace_back(std::move(historicalMetadata)); + } + } + } + + web::json::value ProductMetadata::ToJson_1_0() + { + AICLI_LOG(Repo, Info, << "Creating metadata JSON 1.0 fields"); + + ProductMetadataFields_1_0 fields; + + web::json::value result; + + result[fields.Version] = web::json::value::string(L"1.0"); + result[fields.ProductVersionMin] = AppInstaller::JSON::GetStringValue(ProductVersionMin.ToString()); + result[fields.ProductVersionMax] = AppInstaller::JSON::GetStringValue(ProductVersionMax.ToString()); + + web::json::value metadataArray = web::json::value::array(); + size_t metadataItemIndex = 0; + for (const auto& item : InstallerMetadataMap) + { + web::json::value itemValue; + + itemValue[fields.InstallerHash] = AppInstaller::JSON::GetStringValue(item.first); + itemValue[fields.SubmissionIdentifier] = AppInstaller::JSON::GetStringValue(item.second.SubmissionIdentifier); + + web::json::value appsAndFeaturesArray = web::json::value::array(); + size_t appsAndFeaturesEntryIndex = 0; + for (const auto& entry : item.second.AppsAndFeaturesEntries) + { + web::json::value entryValue; + + AddFieldIfNotEmpty(entryValue, fields.DisplayName, entry.DisplayName); + AddFieldIfNotEmpty(entryValue, fields.Publisher, entry.Publisher); + AddFieldIfNotEmpty(entryValue, fields.DisplayVersion, entry.DisplayVersion); + AddFieldIfNotEmpty(entryValue, fields.ProductCode, entry.ProductCode); + AddFieldIfNotEmpty(entryValue, fields.UpgradeCode, entry.UpgradeCode); + if (entry.InstallerType != Manifest::InstallerTypeEnum::Unknown) + { + entryValue[fields.InstallerType] = AppInstaller::JSON::GetStringValue(Manifest::InstallerTypeToString(entry.InstallerType)); + } + + appsAndFeaturesArray[appsAndFeaturesEntryIndex++] = std::move(entryValue); + } + + itemValue[fields.AppsAndFeaturesEntries] = std::move(appsAndFeaturesArray); + + metadataArray[metadataItemIndex++] = std::move(itemValue); + } + + result[fields.Metadata] = std::move(metadataArray); + + web::json::value historicalArray = web::json::value::array(); + size_t historicalItemIndex = 0; + for (const auto& item : HistoricalMetadataList) + { + web::json::value itemValue; + + itemValue[fields.VersionMin] = AppInstaller::JSON::GetStringValue(item.ProductVersionMin.ToString()); + itemValue[fields.VersionMax] = AppInstaller::JSON::GetStringValue(item.ProductVersionMax.ToString()); + itemValue[fields.Names] = CreateStringArray(item.Names); + itemValue[fields.Publishers] = CreateStringArray(item.Publishers); + itemValue[fields.ProductCodes] = CreateStringArray(item.ProductCodes); + itemValue[fields.UpgradeCodes] = CreateStringArray(item.UpgradeCodes); + + historicalArray[historicalItemIndex++] = std::move(itemValue); + } + + result[fields.Historical] = std::move(historicalArray); + + return result; + } + + bool ProductMetadata::DropOldestHistoricalData() + { + if (HistoricalMetadataList.empty()) + { + return false; + } + + HistoricalMetadataList.pop_back(); + return true; + } + + InstallerMetadataCollectionContext::InstallerMetadataCollectionContext() : + m_correlationData(std::make_unique<Correlation::ARPCorrelationData>()) + {} + + InstallerMetadataCollectionContext::InstallerMetadataCollectionContext(std::unique_ptr<Correlation::ARPCorrelationData> correlationData, const std::wstring& json) : + m_correlationData(std::move(correlationData)) + { + auto threadGlobalsLifetime = InitializeLogging({}); + InitializePreinstallState(json); + } + + std::unique_ptr<InstallerMetadataCollectionContext> InstallerMetadataCollectionContext::FromFile(const std::filesystem::path& file, const std::filesystem::path& logFile) + { + THROW_HR_IF(E_INVALIDARG, file.empty()); + THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND), !std::filesystem::exists(file)); + + std::unique_ptr<InstallerMetadataCollectionContext> result = std::make_unique<InstallerMetadataCollectionContext>(); + auto threadGlobalsLifetime = result->InitializeLogging(logFile); + + AICLI_LOG(Repo, Info, << "Opening InstallerMetadataCollectionContext input file: " << file); + std::ifstream fileStream{ file }; + + result->InitializePreinstallState(ConvertToUTF16(ReadEntireStream(fileStream))); + + return result; + } + + std::unique_ptr<InstallerMetadataCollectionContext> InstallerMetadataCollectionContext::FromURI(std::wstring_view uri, const std::filesystem::path& logFile) + { + THROW_HR_IF(E_INVALIDARG, uri.empty()); + + std::unique_ptr<InstallerMetadataCollectionContext> result = std::make_unique<InstallerMetadataCollectionContext>(); + auto threadGlobalsLifetime = result->InitializeLogging(logFile); + + std::string utf8Uri = ConvertToUTF8(uri); + THROW_HR_IF(E_INVALIDARG, !IsUrlRemote(utf8Uri)); + + AICLI_LOG(Repo, Info, << "Downloading InstallerMetadataCollectionContext input file: " << utf8Uri); + + std::ostringstream jsonStream; + ProgressCallback emptyCallback; + + const int MaxRetryCount = 2; + for (int retryCount = 0; retryCount < MaxRetryCount; ++retryCount) + { + try + { + auto downloadHash = DownloadToStream(utf8Uri, jsonStream, DownloadType::InstallerMetadataCollectionInput, emptyCallback); + break; + } + catch (...) + { + if (retryCount < MaxRetryCount - 1) + { + AICLI_LOG(Repo, Info, << " Downloading InstallerMetadataCollectionContext input failed, waiting a bit and retrying..."); + Sleep(500); + } + else + { + throw; + } + } + } + + result->InitializePreinstallState(ConvertToUTF16(jsonStream.str())); + + return result; + } + + std::unique_ptr<InstallerMetadataCollectionContext> InstallerMetadataCollectionContext::FromJSON(const std::wstring& json, const std::filesystem::path& logFile) + { + THROW_HR_IF(E_INVALIDARG, json.empty()); + + std::unique_ptr<InstallerMetadataCollectionContext> result = std::make_unique<InstallerMetadataCollectionContext>(); + auto threadGlobalsLifetime = result->InitializeLogging(logFile); + result->InitializePreinstallState(json); + + return result; + } + + void InstallerMetadataCollectionContext::Complete(const std::filesystem::path& output) + { + auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); + + THROW_HR_IF(E_INVALIDARG, !output.has_filename()); + + if (output.has_parent_path()) + { + std::filesystem::create_directories(output.parent_path()); + } + + std::ofstream outputStream{ output }; + THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_OPEN_FAILED), !outputStream); + + CompleteWithThreadGlobalsSet(outputStream); + } + + void InstallerMetadataCollectionContext::Complete(std::ostream& output) + { + auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread(); + CompleteWithThreadGlobalsSet(output); + } + + std::wstring InstallerMetadataCollectionContext::Merge(const std::wstring& json, size_t maximumSizeInBytes, const std::filesystem::path& logFile) + { + ThreadLocalStorage::ThreadGlobals threadGlobals; + auto globalsLifetime = InitializeLogging(threadGlobals, logFile); + + AICLI_LOG(Repo, Info, << "Parsing input JSON:\n" << ConvertToUTF8(json)); + + // Parse and validate JSON + try + { + utility::string_t versionFieldName = L"version"; + + web::json::value inputValue = web::json::value::parse(json); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, inputValue.is_null()); + + Version inputVersion = Version{ GetRequiredString(inputValue, versionFieldName) }; + AICLI_LOG(Repo, Info, << "Parsing input JSON version " << inputVersion.ToString()); + + web::json::value mergedResult; + + if (inputVersion.PartAt(0).Integer == 1) + { + mergedResult = Merge_1_0(inputValue, maximumSizeInBytes); + } + else + { + AICLI_LOG(Repo, Error, << "Don't know how to handle version " << inputVersion.ToString()); + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } + + std::wostringstream outputStream; + mergedResult.serialize(outputStream); + + return std::move(outputStream).str(); + } + catch (const web::json::json_exception& exc) + { + AICLI_LOG(Repo, Error, << "Exception parsing input JSON: " << exc.what()); + } + + // We will return within the try or throw a non-json exception, so if we get here it was a json exception. + THROW_HR(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE); + } + + void InstallerMetadataCollectionContext::CompleteWithThreadGlobalsSet(std::ostream& output) + { + web::json::value outputJSON; + + if (!ContainsError()) + { + try + { + // Collect post-install system state + m_correlationData->CapturePostInstallSnapshot(); + + ComputeOutputData(); + + // Construct output JSON + AICLI_LOG(Repo, Info, << "Creating output JSON version for input version " << m_inputVersion.ToString()); + + if (m_inputVersion.PartAt(0).Integer == 1) + { + // We only have one version currently, so use that as long as the major version is 1 + outputJSON = CreateOutputJson_1_0(); + } + else + { + AICLI_LOG(Repo, Error, << "Don't know how to output for version " << m_inputVersion.ToString()); + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } + } + catch (...) + { + CollectErrorDataFromException(std::current_exception()); + } + } + + if (ContainsError()) + { + // We only have one version currently + outputJSON = CreateErrorJson_1_0(); + } + + // Write output + outputJSON.serialize(output); + } + + std::unique_ptr<ThreadLocalStorage::PreviousThreadGlobals> InstallerMetadataCollectionContext::InitializeLogging(ThreadLocalStorage::ThreadGlobals& threadGlobals, const std::filesystem::path& logFile) + { + auto threadGlobalsLifetime = threadGlobals.SetForCurrentThread(); + + Logging::Log().SetLevel(Logging::Level::Info); + Logging::Log().EnableChannel(Logging::Channel::All); + Logging::EnableWilFailureTelemetry(); + Logging::AddTraceLogger(); + + if (!logFile.empty()) + { + Logging::AddFileLogger(logFile); + } + + Logging::Telemetry().SetCaller("installer-metadata-collection"); + Logging::Telemetry().LogStartup(); + + return threadGlobalsLifetime; + } + + std::unique_ptr<ThreadLocalStorage::PreviousThreadGlobals> InstallerMetadataCollectionContext::InitializeLogging(const std::filesystem::path& logFile) + { + return InitializeLogging(m_threadGlobals, logFile); + } + + void InstallerMetadataCollectionContext::InitializePreinstallState(const std::wstring& json) + { + try + { + AICLI_LOG(Repo, Info, << "Parsing input JSON:\n" << ConvertToUTF8(json)); + + // Parse and validate JSON + try + { + utility::string_t versionFieldName = L"version"; + + web::json::value inputValue = web::json::value::parse(json); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, inputValue.is_null()); + + m_inputVersion = Version{ GetRequiredString(inputValue, versionFieldName) }; + AICLI_LOG(Repo, Info, << "Parsing input JSON version " << m_inputVersion.ToString()); + + if (m_inputVersion.PartAt(0).Integer == 1) + { + // We only have one version currently, so use that as long as the major version is 1 + ParseInputJson_1_0(inputValue); + } + else + { + AICLI_LOG(Repo, Error, << "Don't know how to handle version " << m_inputVersion.ToString()); + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } + } + catch (const web::json::json_exception& exc) + { + AICLI_LOG(Repo, Error, << "Exception parsing input JSON: " << exc.what()); + throw; + } + + // Collect pre-install system state + m_correlationData->CapturePreInstallSnapshot(); + } + catch (...) + { + CollectErrorDataFromException(std::current_exception()); + } + } + + void InstallerMetadataCollectionContext::ComputeOutputData() + { + // Copy the metadata from the current; this function takes care of moving data to historical if the submission is new. + m_outputMetadata.CopyFrom(m_currentMetadata, m_submissionIdentifier); + + Correlation::ARPCorrelationResult correlationResult = m_correlationData->CorrelateForNewlyInstalled(m_incomingManifest); + + if (correlationResult.Package) + { + m_outputStatus = OutputStatus::Success; + auto& package = correlationResult.Package; + + // Update min and max versions based on the version of the correlated package + Version packageVersion{ package->GetProperty(PackageVersionProperty::Version) }; + + if (m_outputMetadata.ProductVersionMin.IsEmpty() || packageVersion < m_outputMetadata.ProductVersionMin) + { + m_outputMetadata.ProductVersionMin = packageVersion; + } + + if (m_outputMetadata.ProductVersionMax.IsEmpty() || m_outputMetadata.ProductVersionMax < packageVersion) + { + m_outputMetadata.ProductVersionMax = packageVersion; + } + + // Create the AppsAndFeaturesEntry that we need to add + Manifest::AppsAndFeaturesEntry newEntry; + auto packageMetadata = package->GetMetadata(); + + // TODO: Use some amount of normalization here to prevent things like versions being in the name from bloating the data + newEntry.DisplayName = package->GetProperty(PackageVersionProperty::Name).get(); + newEntry.DisplayVersion = packageVersion.ToString(); + if (packageMetadata.count(PackageVersionMetadata::InstalledType)) + { + newEntry.InstallerType = Manifest::ConvertToInstallerTypeEnum(packageMetadata[PackageVersionMetadata::InstalledType]); + } + auto productCodes = package->GetMultiProperty(PackageVersionMultiProperty::ProductCode); + if (!productCodes.empty()) + { + newEntry.ProductCode = std::move(productCodes[0]).get(); + } + newEntry.Publisher = package->GetProperty(PackageVersionProperty::Publisher).get(); + // TODO: Support upgrade code throughout the code base... + + // Add or update the metadata for the installer hash + auto itr = m_outputMetadata.InstallerMetadataMap.find(m_installerHash); + + if (itr == m_outputMetadata.InstallerMetadataMap.end()) + { + // New entry needed + ProductMetadata::InstallerMetadata newMetadata; + + newMetadata.SubmissionIdentifier = m_submissionIdentifier; + newMetadata.AppsAndFeaturesEntries.emplace_back(std::move(newEntry)); + + m_outputMetadata.InstallerMetadataMap[m_installerHash] = std::move(newMetadata); + } + else + { + // Existing entry for installer hash, add/update the entry + FilterAndAddToEntries(std::move(newEntry), itr->second.AppsAndFeaturesEntries); + } + } + else + { + m_outputStatus = OutputStatus::LowConfidence; + + // TODO: Output diagnostics such as the top 10 entries by confidence. + } + } + + void InstallerMetadataCollectionContext::ParseInputJson_1_0(web::json::value& input) + { + AICLI_LOG(Repo, Info, << "Parsing input JSON 1.0 fields"); + + // Field names + utility::string_t metadataVersionFieldName = L"supportedMetadataVersion"; + utility::string_t metadataFieldName = L"currentMetadata"; + utility::string_t submissionDataFieldName = L"submissionData"; + utility::string_t submissionIdentifierFieldName = L"submissionIdentifier"; + utility::string_t packageDataFieldName = L"packageData"; + utility::string_t installerHashFieldName = L"installerHash"; + utility::string_t defaultLocaleFieldName = L"DefaultLocale"; + utility::string_t localesFieldName = L"Locales"; + + // root fields + m_supportedMetadataVersion = Version{ GetRequiredString(input, metadataVersionFieldName) }; + + auto currentMetadataValue = AppInstaller::JSON::GetJsonValueFromNode(input, metadataFieldName); + if (currentMetadataValue) + { + m_currentMetadata.FromJson(currentMetadataValue.value()); + } + + // submissionData fields + auto submissionDataValue = AppInstaller::JSON::GetJsonValueFromNode(input, submissionDataFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !submissionDataValue); + m_submissionData = submissionDataValue.value(); + + m_submissionIdentifier = GetRequiredString(m_submissionData, submissionIdentifierFieldName); + + // packageData fields + auto packageDataValue = AppInstaller::JSON::GetJsonValueFromNode(input, packageDataFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !packageDataValue); + + m_installerHash = GetRequiredString(packageDataValue.value(), installerHashFieldName); + + // The 1.0 version of input uses the 1.1 version of REST + JSON::ManifestJSONParser parser{ Version{ "1.1" }}; + + { + auto defaultLocaleValue = AppInstaller::JSON::GetJsonValueFromNode(packageDataValue.value(), defaultLocaleFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !defaultLocaleValue); + + auto defaultLocale = parser.DeserializeLocale(defaultLocaleValue.value()); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, + !defaultLocale || + !defaultLocale->Contains(Manifest::Localization::PackageName) || + !defaultLocale->Contains(Manifest::Localization::Publisher)); + + m_incomingManifest.DefaultLocalization = std::move(defaultLocale).value(); + + auto localesArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(packageDataValue.value(), localesFieldName); + if (localesArray) + { + for (const auto& locale : localesArray->get()) + { + auto localization = parser.DeserializeLocale(locale); + if (localization) + { + m_incomingManifest.Localizations.emplace_back(std::move(localization).value()); + } + } + } + } + } + + web::json::value InstallerMetadataCollectionContext::CreateOutputJson_1_0() + { + AICLI_LOG(Repo, Info, << "Setting output JSON 1.0 fields"); + + OutputFields_1_0 fields; + + web::json::value result; + + result[fields.Version] = web::json::value::string(L"1.0"); + result[fields.SubmissionData] = m_submissionData; + result[fields.InstallerHash] = AppInstaller::JSON::GetStringValue(m_installerHash); + + // Limit output status to 1.0 known values + OutputStatus statusToUse = OutputStatus::Unknown; + if (m_outputStatus == OutputStatus::Success || m_outputStatus == OutputStatus::Error || m_outputStatus == OutputStatus::LowConfidence) + { + statusToUse = m_outputStatus; + } + result[fields.Status] = web::json::value::string(ToString(statusToUse)); + + if (m_outputStatus == OutputStatus::Success) + { + result[fields.Metadata] = m_outputMetadata.ToJson(m_supportedMetadataVersion, 0); + } + + result[fields.Diagnostics] = m_outputDiagnostics; + + return result; + } + + utility::string_t InstallerMetadataCollectionContext::ToString(OutputStatus status) + { + switch (status) + { + case OutputStatus::Success: return L"Success"; + case OutputStatus::Error: return L"Error"; + case OutputStatus::LowConfidence: return L"LowConfidence"; + } + + // For both the status value of Unknown and anything else + return L"Unknown"; + } + + bool InstallerMetadataCollectionContext::ContainsError() const + { + return m_outputStatus == OutputStatus::Error; + } + + void InstallerMetadataCollectionContext::CollectErrorDataFromException(std::exception_ptr exception) + { + m_outputStatus = OutputStatus::Error; + + try + { + std::rethrow_exception(exception); + } + catch (const wil::ResultException& re) + { + m_errorHR = re.GetErrorCode(); + m_errorText = GetUserPresentableMessage(re); + } + catch (const winrt::hresult_error& hre) + { + m_errorHR = hre.code(); + m_errorText = GetUserPresentableMessage(hre); + } + catch (const std::exception& e) + { + m_errorHR = E_FAIL; + m_errorText = GetUserPresentableMessage(e); + } + catch (...) + { + m_errorHR = E_UNEXPECTED; + m_errorText = "An unexpected exception type was thrown."; + } + } + + web::json::value InstallerMetadataCollectionContext::CreateErrorJson_1_0() + { + AICLI_LOG(Repo, Info, << "Setting error JSON 1.0 fields"); + + OutputFields_1_0 fields; + + web::json::value result; + + result[fields.Version] = web::json::value::string(L"1.0"); + result[fields.SubmissionData] = m_submissionData; + result[fields.InstallerHash] = AppInstaller::JSON::GetStringValue(m_installerHash); + result[fields.Status] = web::json::value::string(ToString(OutputStatus::Error)); + result[fields.Metadata] = web::json::value::null(); + + web::json::value error; + + error[fields.ErrorHR] = web::json::value::number(static_cast<int64_t>(m_errorHR)); + error[fields.ErrorText] = AppInstaller::JSON::GetStringValue(m_errorText); + + result[fields.Diagnostics] = std::move(error); + + return result; + } + + web::json::value InstallerMetadataCollectionContext::Merge_1_0(web::json::value& input, size_t maximumSizeInBytes) + { + AICLI_LOG(Repo, Info, << "Merging 1.0 input metadatas"); + + utility::string_t metadatasFieldName = L"metadatas"; + + auto metadatasValue = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(input, metadatasFieldName); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !metadatasValue); + + std::vector<ProductMetadata> metadatas; + for (const auto& value : metadatasValue->get()) + { + ProductMetadata current; + current.FromJson(value); + metadatas.emplace_back(std::move(current)); + } + + THROW_HR_IF(E_NOT_SET, metadatas.empty()); + + // Require that all merging values use the same submission + for (const ProductMetadata& metadata : metadatas) + { + const std::string& firstSubmission = metadatas[0].InstallerMetadataMap.begin()->second.SubmissionIdentifier; + const std::string& metadataSubmission = metadata.InstallerMetadataMap.begin()->second.SubmissionIdentifier; + if (firstSubmission != metadataSubmission) + { + AICLI_LOG(Repo, Info, << "Found submission identifier mismatch: " << firstSubmission << " != " << metadataSubmission); + THROW_HR(E_NOT_VALID_STATE); + } + } + + // Do the actual merging + ProductMetadata resultMetadata; + + // The historical data should be the same across the board, so we can just copy the first one. + resultMetadata.HistoricalMetadataList = metadatas[0].HistoricalMetadataList; + + for (const ProductMetadata& metadata : metadatas) + { + // Get the minimum and maximum versions from the individual values + if (resultMetadata.ProductVersionMin.IsEmpty() || metadata.ProductVersionMin < resultMetadata.ProductVersionMin) + { + resultMetadata.ProductVersionMin = metadata.ProductVersionMin; + } + + if (resultMetadata.ProductVersionMax < metadata.ProductVersionMax) + { + resultMetadata.ProductVersionMax = metadata.ProductVersionMax; + } + + if (resultMetadata.SchemaVersion < metadata.SchemaVersion) + { + resultMetadata.SchemaVersion = metadata.SchemaVersion; + } + + for (const auto& installerMetadata : metadata.InstallerMetadataMap) + { + auto itr = resultMetadata.InstallerMetadataMap.find(installerMetadata.first); + if (itr == resultMetadata.InstallerMetadataMap.end()) + { + // Installer hash not in the result, so just copy it + resultMetadata.InstallerMetadataMap.emplace(installerMetadata); + } + else + { + // Merge into existing installer data + for (const auto& targetEntry : installerMetadata.second.AppsAndFeaturesEntries) + { + FilterAndAddToEntries(Manifest::AppsAndFeaturesEntry{ targetEntry }, itr->second.AppsAndFeaturesEntries); + } + } + } + } + + // Convert to JSON + return resultMetadata.ToJson(resultMetadata.SchemaVersion, maximumSizeInBytes); + } +} diff --git a/src/AppInstallerRepositoryCore/ManifestJSONParser.cpp b/src/AppInstallerRepositoryCore/ManifestJSONParser.cpp @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Public/winget/ManifestJSONParser.h" +#include "Rest/Schema/1_0/Json/ManifestDeserializer.h" +#include "Rest/Schema/1_1/Json/ManifestDeserializer.h" + +namespace AppInstaller::Repository::JSON +{ + struct ManifestJSONParser::impl + { + // The deserializer. We only have one lineage (1.0+) right now. + std::unique_ptr<Rest::Schema::V1_0::Json::ManifestDeserializer> m_deserializer; + }; + + ManifestJSONParser::ManifestJSONParser(const Utility::Version& responseSchemaVersion) + { + const auto& parts = responseSchemaVersion.GetParts(); + THROW_HR_IF(E_INVALIDARG, parts.empty()); + + m_pImpl = std::make_unique<impl>(); + + if (parts[0].Integer == 1) + { + if (parts.size() == 1 || parts[1].Integer == 0) + { + m_pImpl->m_deserializer = std::make_unique<Rest::Schema::V1_0::Json::ManifestDeserializer>(); + } + else + { + m_pImpl->m_deserializer = std::make_unique<Rest::Schema::V1_1::Json::ManifestDeserializer>(); + } + } + else + { + THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE)); + } + } + + ManifestJSONParser::ManifestJSONParser(ManifestJSONParser&&) noexcept = default; + ManifestJSONParser& ManifestJSONParser::operator=(ManifestJSONParser&&) noexcept = default; + + ManifestJSONParser::~ManifestJSONParser() = default; + + std::vector<Manifest::Manifest> ManifestJSONParser::Deserialize(const web::json::value& response) const + { + return m_pImpl->m_deserializer->Deserialize(response); + } + + std::vector<Manifest::Manifest> ManifestJSONParser::DeserializeData(const web::json::value& data) const + { + return m_pImpl->m_deserializer->DeserializeData(data); + } + + std::vector<Manifest::AppsAndFeaturesEntry> ManifestJSONParser::DeserializeAppsAndFeaturesEntries(const web::json::array& data) const + { + return m_pImpl->m_deserializer->DeserializeAppsAndFeaturesEntries(data); + } + + std::optional<Manifest::ManifestLocalization> ManifestJSONParser::DeserializeLocale(const web::json::value& locale) const + { + return m_pImpl->m_deserializer->DeserializeLocale(locale); + } + +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/ConfigurableTestSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/ConfigurableTestSourceFactory.cpp @@ -3,7 +3,7 @@ #include "pch.h" #include "Microsoft/ConfigurableTestSourceFactory.h" -#include <json.h> +#include <json/json.h> using namespace std::string_literals; using namespace std::string_view_literals; diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp @@ -121,7 +121,6 @@ namespace AppInstaller::Repository::Microsoft const int MaxRetryCount = 2; for (int retryCount = 0; retryCount < MaxRetryCount; ++retryCount) { - bool success = false; try { auto downloadHash = Utility::DownloadToStream(fullPath, manifestStream, Utility::DownloadType::Manifest, emptyCallback, !expectedHash.empty()); @@ -132,7 +131,7 @@ namespace AppInstaller::Repository::Microsoft THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE); } - success = true; + break; } catch (...) { @@ -146,11 +145,6 @@ namespace AppInstaller::Repository::Microsoft throw; } } - - if (success) - { - break; - } } std::string manifestContents = manifestStream.str(); diff --git a/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h b/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h @@ -2,6 +2,13 @@ // Licensed under the MIT License. #pragma once +#include <winget/LocIndependent.h> +#include <winget/RepositorySource.h> + +#include <memory> +#include <utility> +#include <vector> + namespace AppInstaller { namespace Manifest @@ -26,7 +33,7 @@ namespace AppInstaller::Repository::Correlation // Struct holding all the data from an ARP entry we use for the correlation struct ARPEntry { - ARPEntry(std::shared_ptr<AppInstaller::Repository::IPackage> entry, bool isNewOrUpdated) : Entry(entry), IsNewOrUpdated(isNewOrUpdated) {} + ARPEntry(std::shared_ptr<AppInstaller::Repository::IPackage> entry, bool isNewOrUpdated) : Entry(std::move(entry)), IsNewOrUpdated(isNewOrUpdated) {} // Data found in the ARP entry std::shared_ptr<AppInstaller::Repository::IPackage> Entry; @@ -63,14 +70,6 @@ namespace AppInstaller::Repository::Correlation #endif }; - // Finds the ARP entry in the ARP source that matches a newly installed package. - // Takes the package manifest, a snapshot of the ARP before the installation, and the current ARP source. - // Returns the entry in the ARP source, or nullptr if there was no match, plus some stats about the correlation. - ARPCorrelationResult FindARPEntryForNewlyInstalledPackage( - const AppInstaller::Manifest::Manifest& manifest, - const std::vector<ARPEntrySnapshot>& arpSnapshot, - AppInstaller::Repository::Source& arpSource); - std::shared_ptr<AppInstaller::Repository::IPackageVersion> FindARPEntryForNewlyInstalledPackageWithHeuristics( const AppInstaller::Manifest::Manifest& manifest, const std::vector<ARPEntry>& arpEntries); @@ -79,4 +78,28 @@ namespace AppInstaller::Repository::Correlation const AppInstaller::Manifest::Manifest& manifest, const std::vector<ARPEntry>& arpEntries, IARPMatchConfidenceAlgorithm& algorithm); -}- \ No newline at end of file + + // Holds data needed for ARP correlation, as well as functions to run correlation on the collected data. + struct ARPCorrelationData + { + ARPCorrelationData() = default; + virtual ~ARPCorrelationData() = default; + + // Captures the ARP state before the package installation. + void CapturePreInstallSnapshot(); + + // Captures the ARP state differences after the package installation. + void CapturePostInstallSnapshot(); + + // Correlates the given manifest against the data previously collected with capture calls. + virtual ARPCorrelationResult CorrelateForNewlyInstalled(const Manifest::Manifest& manifest); + + const std::vector<ARPEntrySnapshot>& GetPreInstallSnapshot() const { return m_preInstallSnapshot; } + + private: + std::vector<ARPEntrySnapshot> m_preInstallSnapshot; + + Source m_postInstallSnapshotSource; + std::vector<Correlation::ARPEntry> m_postInstallSnapshot; + }; +} diff --git a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h @@ -0,0 +1,164 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once + +#include <AppInstallerVersions.h> +#include <winget/Manifest.h> +#include <winget/JsonUtil.h> +#include <winget/ThreadGlobals.h> +#include <winget/ARPCorrelation.h> + +#include <filesystem> +#include <map> +#include <memory> +#include <optional> +#include <ostream> +#include <string> +#include <string_view> + +namespace AppInstaller::Repository::Metadata +{ + // The overall metadata that we collect. + struct ProductMetadata + { + ProductMetadata() = default; + + // Removes all stored data. + void Clear(); + + // Load the metadata from an existing JSON blob. + void FromJson(const web::json::value& json); + + // Create a JSON value for the metadata using the given schema version. + web::json::value ToJson(const Utility::Version& schemaVersion, size_t maximumSizeInBytes); + + // Copies the metadata from the source. If the given submission identifier does not match + // the source, it's data is moved to historical. + void CopyFrom(const ProductMetadata& source, std::string_view submissionIdentifier); + + // The installer specific metadata that we collect. + struct InstallerMetadata + { + std::string SubmissionIdentifier; + std::vector<Manifest::AppsAndFeaturesEntry> AppsAndFeaturesEntries; + }; + + // Metadata from previous product revisions. + struct HistoricalMetadata + { + Utility::Version ProductVersionMin; + Utility::Version ProductVersionMax; + std::set<std::string> Names; + std::set<std::string> Publishers; + std::set<std::string> ProductCodes; + std::set<std::string> UpgradeCodes; + }; + + Utility::Version SchemaVersion; + Utility::Version ProductVersionMin; + Utility::Version ProductVersionMax; + // Map from installer hash to metadata + std::map<std::string, InstallerMetadata> InstallerMetadataMap; + std::vector<HistoricalMetadata> HistoricalMetadataList; + + private: + void FromJson_1_0(const web::json::value& json); + web::json::value ToJson_1_0(); + + // Removes the historical data with the oldest version. + // Returns true if something was removed; false it not. + bool DropOldestHistoricalData(); + }; + + // Contains the functions and data used for collecting metadata from installers. + struct InstallerMetadataCollectionContext + { + InstallerMetadataCollectionContext(); + InstallerMetadataCollectionContext(std::unique_ptr<Correlation::ARPCorrelationData> correlationData, const std::wstring& json); + + InstallerMetadataCollectionContext(const InstallerMetadataCollectionContext&) = delete; + InstallerMetadataCollectionContext& operator=(const InstallerMetadataCollectionContext&) = delete; + + InstallerMetadataCollectionContext(InstallerMetadataCollectionContext&&) = default; + InstallerMetadataCollectionContext& operator=(InstallerMetadataCollectionContext&&) = default; + + // Create from various forms of JSON input to prevent type collisions on constructor. + static std::unique_ptr<InstallerMetadataCollectionContext> FromFile(const std::filesystem::path& file, const std::filesystem::path& logFile); + static std::unique_ptr<InstallerMetadataCollectionContext> FromURI(std::wstring_view uri, const std::filesystem::path& logFile); + static std::unique_ptr<InstallerMetadataCollectionContext> FromJSON(const std::wstring& json, const std::filesystem::path& logFile); + + // Completes the collection, writing to the given location. + void Complete(const std::filesystem::path& output); + + // Completes the collection, writing to the given location. + void Complete(std::ostream& output); + + static std::wstring Merge(const std::wstring& json, size_t maximumSizeInBytes, const std::filesystem::path& logFile); + + private: + // Initializes the context runtime, including the log file if provided. + static std::unique_ptr<ThreadLocalStorage::PreviousThreadGlobals> InitializeLogging(ThreadLocalStorage::ThreadGlobals& threadGlobals, const std::filesystem::path& logFile); + std::unique_ptr<ThreadLocalStorage::PreviousThreadGlobals> InitializeLogging(const std::filesystem::path& logFile); + + // Sets the collection context input and the preinstall state. + void InitializePreinstallState(const std::wstring& json); + + // Creates the output ProductMetadata and diagnostics objects for output + void ComputeOutputData(); + + // Callers should set the thread globals before calling this. + void CompleteWithThreadGlobalsSet(std::ostream& output); + + // Parse version 1.0 of input JSON + void ParseInputJson_1_0(web::json::value& input); + + // Create version 1.0 of output JSON + web::json::value CreateOutputJson_1_0(); + + // Determines whether an error has occurred in the context. + bool ContainsError() const; + + // Collects information from the exception for error reporting. + void CollectErrorDataFromException(std::exception_ptr exception); + + // Create version 1.0 of error JSON + web::json::value CreateErrorJson_1_0(); + + // Merge using merge input version 1.0 + static web::json::value Merge_1_0(web::json::value& input, size_t maximumSizeInBytes); + + ThreadLocalStorage::ThreadGlobals m_threadGlobals; + + // Parsed input + Utility::Version m_inputVersion; + Utility::Version m_supportedMetadataVersion; + ProductMetadata m_currentMetadata; + web::json::value m_submissionData; + std::string m_submissionIdentifier; + std::string m_installerHash; + Manifest::Manifest m_incomingManifest; + + std::unique_ptr<Correlation::ARPCorrelationData> m_correlationData; + + // Output data + enum class OutputStatus + { + // Version 1.0 status values + Unknown, + Success, + Error, + LowConfidence, + }; + + // Convert status to a JSON string value + static utility::string_t ToString(OutputStatus status); + + OutputStatus m_outputStatus = OutputStatus::Unknown; + ProductMetadata m_outputMetadata; + web::json::value m_outputDiagnostics; + + // Error data storage + HRESULT m_errorHR = S_OK; + std::string m_errorText; + }; +} diff --git a/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h b/src/AppInstallerRepositoryCore/Public/winget/ManifestJSONParser.h @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include <AppInstallerVersions.h> +#include <winget/JsonUtil.h> +#include <winget/Manifest.h> + +#include <memory> +#include <vector> + +namespace AppInstaller::Repository::JSON +{ + // Exposes functions for parsing JSON REST responses to manifest requests. + struct ManifestJSONParser + { + ManifestJSONParser(const Utility::Version& responseSchemaVersion); + + ManifestJSONParser(const ManifestJSONParser&) = delete; + ManifestJSONParser& operator=(const ManifestJSONParser&) = delete; + + ManifestJSONParser(ManifestJSONParser&&) noexcept; + ManifestJSONParser& operator=(ManifestJSONParser&&) noexcept; + + ~ManifestJSONParser(); + + // Deserializes the manifests from the REST response object root. + // May potentially contain multiple versions of the same package. + std::vector<Manifest::Manifest> Deserialize(const web::json::value& response) const; + + // Deserializes the manifests from the Data field of the REST response object. + // May potentially contain multiple versions of the same package. + std::vector<Manifest::Manifest> DeserializeData(const web::json::value& data) const; + + // Deserializes the AppsAndFeaturesEntries node, returning the set of values below it. + std::vector<Manifest::AppsAndFeaturesEntry> DeserializeAppsAndFeaturesEntries(const web::json::array& data) const; + + // Deserializes the locale node; returning an object if a proper locale was found. + std::optional<Manifest::ManifestLocalization> DeserializeLocale(const web::json::value& locale) const; + + private: + struct impl; + std::unique_ptr<impl> m_pImpl; + }; +} diff --git a/src/AppInstallerRepositoryCore/Rest/RestClient.cpp b/src/AppInstallerRepositoryCore/Rest/RestClient.cpp @@ -5,8 +5,8 @@ #include "Rest/Schema/1_0/Interface.h" #include "Rest/Schema/1_1/Interface.h" #include "Rest/Schema/HttpClientHelper.h" +#include <winget/JsonUtil.h> #include "Rest/Schema/InformationResponseDeserializer.h" -#include "Rest/Schema/JsonHelper.h" #include "Rest/Schema/CommonRestConstants.h" #include "Rest/Schema/RestHelper.h" @@ -34,7 +34,7 @@ namespace AppInstaller::Repository::Rest THROW_HR_IF(APPINSTALLER_CLI_ERROR_CUSTOMHEADER_EXCEEDS_MAXLENGTH, customHeader.value().size() > WindowsPackageManagerHeaderMaxLength); std::unordered_map<utility::string_t, utility::string_t> headers; - headers.emplace(JsonHelper::GetUtilityString(WindowsPackageManagerHeader), JsonHelper::GetUtilityString(customHeader.value())); + headers.emplace(JSON::GetUtilityString(WindowsPackageManagerHeader), JSON::GetUtilityString(customHeader.value())); return headers; } } @@ -68,7 +68,7 @@ namespace AppInstaller::Repository::Rest const utility::string_t& restApi, const std::unordered_map<utility::string_t, utility::string_t>& additionalHeaders, const HttpClientHelper& clientHelper) { // Call information endpoint - utility::string_t endpoint = RestHelper::AppendPathToUri(restApi, JsonHelper::GetUtilityString(InformationGetEndpoint)); + utility::string_t endpoint = RestHelper::AppendPathToUri(restApi, JSON::GetUtilityString(InformationGetEndpoint)); std::optional<web::json::value> response = clientHelper.HandleGet(endpoint, additionalHeaders); THROW_HR_IF(APPINSTALLER_CLI_ERROR_UNSUPPORTED_RESTSOURCE, !response); diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer.h @@ -3,33 +3,38 @@ #pragma once #include <winget/Manifest.h> #include <cpprest/json.h> -#include "Rest\Schema\JsonHelper.h" +#include <winget/JsonUtil.h> namespace AppInstaller::Repository::Rest::Schema::V1_0::Json { // Manifest Deserializer. struct ManifestDeserializer { - // Gets the manifest from the given json object - std::vector<Manifest::Manifest> Deserialize(const web::json::value& dataJsonObject) const; + // Gets the manifest from the given json object received from a REST request + std::vector<Manifest::Manifest> Deserialize(const web::json::value& responseJsonObject) const; + + // Gets the manifest from the given json Data field + std::vector<Manifest::Manifest> DeserializeData(const web::json::value& dataJsonObject) const; + + // Deserializes the AppsAndFeaturesEntries node, returning the set of values below it. + virtual std::vector<Manifest::AppsAndFeaturesEntry> DeserializeAppsAndFeaturesEntries(const web::json::array& entries) const; + + // Deserializes the locale; requires that the PackageLocale be set to return an object. + virtual std::optional<Manifest::ManifestLocalization> DeserializeLocale(const web::json::value& localeJsonObject) const; protected: template <Manifest::Localization L> inline void TryParseStringLocaleField(Manifest::ManifestLocalization& manifestLocale, const web::json::value& localeJsonObject, std::string_view localeJsonFieldName) const { - auto value = JsonHelper::GetRawStringValueFromJsonNode(localeJsonObject, JsonHelper::GetUtilityString(localeJsonFieldName)); + auto value = AppInstaller::JSON::GetRawStringValueFromJsonNode(localeJsonObject, AppInstaller::JSON::GetUtilityString(localeJsonFieldName)); - if (JsonHelper::IsValidNonEmptyStringValue(value)) + if (AppInstaller::JSON::IsValidNonEmptyStringValue(value)) { manifestLocale.Add<L>(value.value()); } } - std::optional<std::vector<Manifest::Manifest>> DeserializeVersion(const web::json::value& dataJsonObject) const; - - virtual std::optional<Manifest::ManifestLocalization> DeserializeLocale(const web::json::value& localeJsonObject) const; - virtual std::optional<Manifest::ManifestInstaller> DeserializeInstaller(const web::json::value& installerJsonObject) const; std::optional<Manifest::DependencyList> DeserializeDependency(const web::json::value& dependenciesJsonObject) const; diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/ManifestDeserializer_1_0.cpp @@ -5,7 +5,7 @@ #include "Rest/Schema/IRestClient.h" #include "Rest/Schema/HttpClientHelper.h" #include "ManifestDeserializer.h" -#include "Rest/Schema/JsonHelper.h" +#include <winget/JsonUtil.h> #include "Rest/Schema/CommonRestConstants.h" using namespace AppInstaller::Manifest; @@ -89,158 +89,165 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json const web::json::value& switchesJsonObject, std::string_view switchJsonFieldName) { - auto value = JsonHelper::GetRawStringValueFromJsonNode(switchesJsonObject, JsonHelper::GetUtilityString(switchJsonFieldName)); + auto value = JSON::GetRawStringValueFromJsonNode(switchesJsonObject, JSON::GetUtilityString(switchJsonFieldName)); - if (JsonHelper::IsValidNonEmptyStringValue(value)) + if (JSON::IsValidNonEmptyStringValue(value)) { installerSwitches[switchType] = value.value(); } } } - std::vector<Manifest::Manifest> ManifestDeserializer::Deserialize(const web::json::value& dataJsonObject) const + std::vector<Manifest::Manifest> ManifestDeserializer::Deserialize(const web::json::value& responseJsonObject) const { - // Get manifest from json output. - std::optional<std::vector<Manifest::Manifest>> manifests = DeserializeVersion(dataJsonObject); - - THROW_HR_IF(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA, !manifests); - - return manifests.value(); - } - - std::optional<std::vector<Manifest::Manifest>> ManifestDeserializer::DeserializeVersion(const web::json::value& dataJsonObject) const - { - if (dataJsonObject.is_null()) + if (responseJsonObject.is_null()) { AICLI_LOG(Repo, Error, << "Missing json object."); - return {}; + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); } - std::vector<Manifest::Manifest> manifests; try { std::optional<std::reference_wrapper<const web::json::value>> manifestObject = - JsonHelper::GetJsonValueFromNode(dataJsonObject, JsonHelper::GetUtilityString(Data)); + JSON::GetJsonValueFromNode(responseJsonObject, JSON::GetUtilityString(Data)); if (!manifestObject || manifestObject.value().get().is_null()) { AICLI_LOG(Repo, Verbose, << "No manifest results returned."); - return manifests; + return {}; } - auto& manifestJsonObject = manifestObject.value().get(); - std::optional<std::string> id = JsonHelper::GetRawStringValueFromJsonNode(manifestJsonObject, JsonHelper::GetUtilityString(PackageIdentifier)); - if (!JsonHelper::IsValidNonEmptyStringValue(id)) + return DeserializeData(manifestObject.value()); + } + catch (const wil::ResultException&) + { + throw; + } + catch (const std::exception& e) + { + AICLI_LOG(Repo, Error, << "Error encountered while deserializing manifest. Reason: " << e.what()); + } + catch (...) + { + AICLI_LOG(Repo, Error, << "Error encountered while deserializing manifest..."); + } + + // If we make it here, there was an exception above that we didn't throw. + // This will convert it into our standard error. + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); + } + + std::vector<Manifest::Manifest> ManifestDeserializer::DeserializeData(const web::json::value& dataJsonObject) const + { + THROW_HR_IF(E_INVALIDARG, dataJsonObject.is_null()); + + std::vector<Manifest::Manifest> manifests; + + std::optional<std::string> id = JSON::GetRawStringValueFromJsonNode(dataJsonObject, JSON::GetUtilityString(PackageIdentifier)); + if (!JSON::IsValidNonEmptyStringValue(id)) + { + AICLI_LOG(Repo, Error, << "Missing package identifier."); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); + } + + std::optional<std::reference_wrapper<const web::json::array>> versions = JSON::GetRawJsonArrayFromJsonNode(dataJsonObject, JSON::GetUtilityString(Versions)); + if (!versions || versions.value().get().size() == 0) + { + AICLI_LOG(Repo, Error, << "Missing versions in package: " << id.value()); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); + } + + for (auto& versionItem : versions.value().get()) + { + Manifest::Manifest manifest; + manifest.Id = id.value(); + + std::optional<std::string> packageVersion = JSON::GetRawStringValueFromJsonNode(versionItem, JSON::GetUtilityString(PackageVersion)); + if (!JSON::IsValidNonEmptyStringValue(packageVersion)) { - AICLI_LOG(Repo, Error, << "Missing package identifier."); - return {}; + AICLI_LOG(Repo, Error, << "Missing package version in package: " << manifest.Id); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); } + manifest.Version = std::move(packageVersion.value()); + + manifest.Channel = JSON::GetRawStringValueFromJsonNode(versionItem, JSON::GetUtilityString(Channel)).value_or(""); - std::optional<std::reference_wrapper<const web::json::array>> versions = JsonHelper::GetRawJsonArrayFromJsonNode(manifestJsonObject, JsonHelper::GetUtilityString(Versions)); - if (!versions || versions.value().get().size() == 0) + // Default locale + std::optional<std::reference_wrapper<const web::json::value>> defaultLocale = + JSON::GetJsonValueFromNode(versionItem, JSON::GetUtilityString(DefaultLocale)); + if (!defaultLocale) { - AICLI_LOG(Repo, Error, << "Missing versions in package: " << id.value()); - return {}; + AICLI_LOG(Repo, Error, << "Missing default locale in package: " << manifest.Id); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); } - - const web::json::array versionNodes = versions.value().get(); - for (auto& versionItem : versionNodes) + else { - Manifest::Manifest manifest; - manifest.Id = id.value(); - - std::optional<std::string> packageVersion = JsonHelper::GetRawStringValueFromJsonNode(versionItem, JsonHelper::GetUtilityString(PackageVersion)); - if (!JsonHelper::IsValidNonEmptyStringValue(packageVersion)) + std::optional<Manifest::ManifestLocalization> defaultLocaleObject = DeserializeLocale(defaultLocale.value().get()); + if (!defaultLocaleObject) { - AICLI_LOG(Repo, Error, << "Missing package version in package: " << manifest.Id); - return {}; + AICLI_LOG(Repo, Error, << "Missing default locale in package: " << manifest.Id); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); } - manifest.Version = std::move(packageVersion.value()); - - manifest.Channel = JsonHelper::GetRawStringValueFromJsonNode(versionItem, JsonHelper::GetUtilityString(Channel)).value_or(""); - // Default locale - std::optional<std::reference_wrapper<const web::json::value>> defaultLocale = - JsonHelper::GetJsonValueFromNode(versionItem, JsonHelper::GetUtilityString(DefaultLocale)); - if (!defaultLocale) + if (!defaultLocaleObject.value().Contains(Manifest::Localization::PackageName) || + !defaultLocaleObject.value().Contains(Manifest::Localization::Publisher) || + !defaultLocaleObject.value().Contains(Manifest::Localization::ShortDescription)) { - AICLI_LOG(Repo, Error, << "Missing default locale in package: " << manifest.Id); - return {}; + AICLI_LOG(Repo, Error, << "Missing PackageName, Publisher or ShortDescription in default locale: " << manifest.Id); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); } - else - { - std::optional<Manifest::ManifestLocalization> defaultLocaleObject = DeserializeLocale(defaultLocale.value().get()); - if (!defaultLocaleObject) - { - AICLI_LOG(Repo, Error, << "Missing default locale in package: " << manifest.Id); - return {}; - } - - if (!defaultLocaleObject.value().Contains(Manifest::Localization::PackageName) || - !defaultLocaleObject.value().Contains(Manifest::Localization::Publisher) || - !defaultLocaleObject.value().Contains(Manifest::Localization::ShortDescription)) - { - AICLI_LOG(Repo, Error, << "Missing PackageName, Publisher or ShortDescription in default locale: " << manifest.Id); - return {}; - } - manifest.DefaultLocalization = std::move(defaultLocaleObject.value()); + manifest.DefaultLocalization = std::move(defaultLocaleObject.value()); - // Moniker is in Default locale - manifest.Moniker = JsonHelper::GetRawStringValueFromJsonNode(defaultLocale.value().get(), JsonHelper::GetUtilityString(Moniker)).value_or(""); - } + // Moniker is in Default locale + manifest.Moniker = JSON::GetRawStringValueFromJsonNode(defaultLocale.value().get(), JSON::GetUtilityString(Moniker)).value_or(""); + } - // Installers - std::optional<std::reference_wrapper<const web::json::array>> installers = JsonHelper::GetRawJsonArrayFromJsonNode(versionItem, JsonHelper::GetUtilityString(Installers)); - if (!installers || installers.value().get().size() == 0) - { - AICLI_LOG(Repo, Error, << "Missing installers in package: " << manifest.Id); - return {}; - } + // Installers + std::optional<std::reference_wrapper<const web::json::array>> installers = JSON::GetRawJsonArrayFromJsonNode(versionItem, JSON::GetUtilityString(Installers)); + if (!installers || installers.value().get().size() == 0) + { + AICLI_LOG(Repo, Error, << "Missing installers in package: " << manifest.Id); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); + } - for (auto& installer : installers.value().get()) + for (auto& installer : installers.value().get()) + { + std::optional<Manifest::ManifestInstaller> installerObject = DeserializeInstaller(installer); + if (installerObject) { - std::optional<Manifest::ManifestInstaller> installerObject = DeserializeInstaller(installer); - if (installerObject) - { - manifest.Installers.emplace_back(std::move(installerObject.value())); - } + manifest.Installers.emplace_back(std::move(installerObject.value())); } + } - if (manifest.Installers.size() == 0) - { - AICLI_LOG(Repo, Error, << "Missing valid installers in package: " << manifest.Id); - return {}; - } + if (manifest.Installers.size() == 0) + { + AICLI_LOG(Repo, Error, << "Missing valid installers in package: " << manifest.Id); + THROW_HR(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_DATA); + } - // Other locales - std::optional<std::reference_wrapper<const web::json::array>> locales = JsonHelper::GetRawJsonArrayFromJsonNode(versionItem, JsonHelper::GetUtilityString(Locales)); - if (locales) + // Other locales + std::optional<std::reference_wrapper<const web::json::array>> locales = JSON::GetRawJsonArrayFromJsonNode(versionItem, JSON::GetUtilityString(Locales)); + if (locales) + { + for (auto& locale : locales.value().get()) { - for (auto& locale : locales.value().get()) + std::optional<Manifest::ManifestLocalization> localeObject = DeserializeLocale(locale); + if (localeObject) { - std::optional<Manifest::ManifestLocalization> localeObject = DeserializeLocale(locale); - if (localeObject) - { - manifest.Localizations.emplace_back(std::move(localeObject.value())); - } + manifest.Localizations.emplace_back(std::move(localeObject.value())); } } - - manifests.emplace_back(std::move(manifest)); } - return manifests; - } - catch (const std::exception& e) - { - AICLI_LOG(Repo, Error, << "Error encountered while deserializing manifest. Reason: " << e.what()); - } - catch (...) - { - AICLI_LOG(Repo, Error, << "Error encountered while deserializing manifest..."); + manifests.emplace_back(std::move(manifest)); } + return manifests; + } + + std::vector<Manifest::AppsAndFeaturesEntry> ManifestDeserializer::DeserializeAppsAndFeaturesEntries(const web::json::array&) const + { return {}; } @@ -252,8 +259,8 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } Manifest::ManifestLocalization locale; - std::optional<std::string> packageLocale = JsonHelper::GetRawStringValueFromJsonNode(localeJsonObject, JsonHelper::GetUtilityString(PackageLocale)); - if (!JsonHelper::IsValidNonEmptyStringValue(packageLocale)) + std::optional<std::string> packageLocale = JSON::GetRawStringValueFromJsonNode(localeJsonObject, JSON::GetUtilityString(PackageLocale)); + if (!JSON::IsValidNonEmptyStringValue(packageLocale)) { AICLI_LOG(Repo, Error, << "Missing package locale."); return {}; @@ -274,7 +281,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json TryParseStringLocaleField<Manifest::Localization::CopyrightUrl>(locale, localeJsonObject, CopyrightUrl); TryParseStringLocaleField<Manifest::Localization::Description>(locale, localeJsonObject, Description); - auto tags = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(localeJsonObject, JsonHelper::GetUtilityString(Tags))); + auto tags = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(localeJsonObject, JSON::GetUtilityString(Tags))); if (!tags.empty()) { locale.Add<AppInstaller::Manifest::Localization::Tags>(tags); @@ -292,38 +299,38 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json Manifest::ManifestInstaller installer; - installer.Url = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallerUrl)).value_or(""); + installer.Url = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallerUrl)).value_or(""); - std::optional<std::string> sha256 = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallerSha256)); - if (JsonHelper::IsValidNonEmptyStringValue(sha256)) + std::optional<std::string> sha256 = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallerSha256)); + if (JSON::IsValidNonEmptyStringValue(sha256)) { installer.Sha256 = Utility::SHA256::ConvertToBytes(sha256.value()); } - std::optional<std::string> arch = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(Architecture)); - if (!JsonHelper::IsValidNonEmptyStringValue(arch)) + std::optional<std::string> arch = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(Architecture)); + if (!JSON::IsValidNonEmptyStringValue(arch)) { AICLI_LOG(Repo, Error, << "Missing installer architecture."); return {}; } installer.Arch = Utility::ConvertToArchitectureEnum(arch.value()); - std::optional<std::string> installerType = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallerType)); - if (!JsonHelper::IsValidNonEmptyStringValue(installerType)) + std::optional<std::string> installerType = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallerType)); + if (!JSON::IsValidNonEmptyStringValue(installerType)) { AICLI_LOG(Repo, Error, << "Missing installer type."); return {}; } installer.InstallerType = ConvertToInstallerType(installerType.value()); - installer.Locale = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallerLocale)).value_or(""); + installer.Locale = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallerLocale)).value_or(""); // platform - std::optional<std::reference_wrapper<const web::json::array>> platforms = JsonHelper::GetRawJsonArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(Platform)); + std::optional<std::reference_wrapper<const web::json::array>> platforms = JSON::GetRawJsonArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(Platform)); if (platforms) { for (auto& platform : platforms.value().get()) { - std::optional<std::string> platformValue = JsonHelper::GetRawStringValueFromJsonValue(platform); + std::optional<std::string> platformValue = JSON::GetRawStringValueFromJsonValue(platform); if (platformValue) { installer.Platform.emplace_back(Manifest::ConvertToPlatformEnum(platformValue.value())); @@ -331,26 +338,26 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } } - installer.MinOSVersion = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(MinimumOSVersion)).value_or(""); - std::optional<std::string> scope = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(Scope)); + installer.MinOSVersion = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(MinimumOSVersion)).value_or(""); + std::optional<std::string> scope = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(Scope)); if (scope) { installer.Scope = Manifest::ConvertToScopeEnum(scope.value()); } - std::optional<std::string> signatureSha256 = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(SignatureSha256)); + std::optional<std::string> signatureSha256 = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(SignatureSha256)); if (signatureSha256) { installer.SignatureSha256 = Utility::SHA256::ConvertToBytes(signatureSha256.value()); } // Install modes - std::optional<std::reference_wrapper<const web::json::array>> installModes = JsonHelper::GetRawJsonArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallModes)); + std::optional<std::reference_wrapper<const web::json::array>> installModes = JSON::GetRawJsonArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallModes)); if (installModes) { for (auto& mode : installModes.value().get()) { - std::optional<std::string> modeObject = JsonHelper::GetRawStringValueFromJsonValue(mode); + std::optional<std::string> modeObject = JSON::GetRawStringValueFromJsonValue(mode); if (modeObject) { installer.InstallModes.emplace_back(Manifest::ConvertToInstallModeEnum(modeObject.value())); @@ -361,7 +368,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json // Installer Switches installer.Switches = Manifest::GetDefaultKnownSwitches(installer.InstallerType); std::optional<std::reference_wrapper<const web::json::value>> switches = - JsonHelper::GetJsonValueFromNode(installerJsonObject, JsonHelper::GetUtilityString(InstallerSwitches)); + JSON::GetJsonValueFromNode(installerJsonObject, JSON::GetUtilityString(InstallerSwitches)); if (switches) { const auto& installerSwitches = switches.value().get(); @@ -375,12 +382,12 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } // Installer SuccessCodes - std::optional<std::reference_wrapper<const web::json::array>> installSuccessCodes = JsonHelper::GetRawJsonArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallerSuccessCodes)); + std::optional<std::reference_wrapper<const web::json::array>> installSuccessCodes = JSON::GetRawJsonArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallerSuccessCodes)); if (installSuccessCodes) { for (auto& code : installSuccessCodes.value().get()) { - std::optional<int> codeValue = JsonHelper::GetRawIntValueFromJsonValue(code); + std::optional<int> codeValue = JSON::GetRawIntValueFromJsonValue(code); if (codeValue) { installer.InstallerSuccessCodes.emplace_back(std::move(codeValue.value())); @@ -388,19 +395,19 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } } - std::optional<std::string> updateBehavior = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(UpgradeBehavior)); + std::optional<std::string> updateBehavior = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(UpgradeBehavior)); if (updateBehavior) { installer.UpdateBehavior = Manifest::ConvertToUpdateBehaviorEnum(updateBehavior.value()); } - installer.Commands = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(Commands))); - installer.Protocols = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(Protocols))); - installer.FileExtensions = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(FileExtensions))); + installer.Commands = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(Commands))); + installer.Protocols = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(Protocols))); + installer.FileExtensions = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(FileExtensions))); // Dependencies std::optional<std::reference_wrapper<const web::json::value>> dependenciesObject = - JsonHelper::GetJsonValueFromNode(installerJsonObject, JsonHelper::GetUtilityString(Dependencies)); + JSON::GetJsonValueFromNode(installerJsonObject, JSON::GetUtilityString(Dependencies)); if (dependenciesObject) { std::optional<Manifest::DependencyList> dependencyList = DeserializeDependency(dependenciesObject.value().get()); @@ -410,10 +417,10 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } } - installer.PackageFamilyName = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(PackageFamilyName)).value_or(""); - installer.ProductCode = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(ProductCode)).value_or(""); - installer.Capabilities = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(Capabilities))); - installer.RestrictedCapabilities = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(RestrictedCapabilities))); + installer.PackageFamilyName = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(PackageFamilyName)).value_or(""); + installer.ProductCode = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(ProductCode)).value_or(""); + installer.Capabilities = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(Capabilities))); + installer.RestrictedCapabilities = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(RestrictedCapabilities))); return installer; } @@ -427,34 +434,34 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json Manifest::DependencyList dependencyList; - auto wfIds = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(dependenciesObject, JsonHelper::GetUtilityString(WindowsFeatures))); + auto wfIds = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(dependenciesObject, JSON::GetUtilityString(WindowsFeatures))); for (auto&& id : wfIds) { dependencyList.Add(Dependency(DependencyType::WindowsFeature, std::move(id))); }; - const auto& wlIds = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(dependenciesObject, JsonHelper::GetUtilityString(WindowsLibraries))); + const auto& wlIds = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(dependenciesObject, JSON::GetUtilityString(WindowsLibraries))); for (auto id : wlIds) { dependencyList.Add(Dependency(DependencyType::WindowsLibrary, id)); }; - const auto& extIds = ConvertToManifestStringArray(JsonHelper::GetRawStringArrayFromJsonNode(dependenciesObject, JsonHelper::GetUtilityString(ExternalDependencies))); + const auto& extIds = ConvertToManifestStringArray(JSON::GetRawStringArrayFromJsonNode(dependenciesObject, JSON::GetUtilityString(ExternalDependencies))); for (auto id : extIds) { dependencyList.Add(Dependency(DependencyType::External, id)); }; // Package Dependencies - std::optional<std::reference_wrapper<const web::json::array>> packageDependencies = JsonHelper::GetRawJsonArrayFromJsonNode(dependenciesObject, JsonHelper::GetUtilityString(PackageDependencies)); + std::optional<std::reference_wrapper<const web::json::array>> packageDependencies = JSON::GetRawJsonArrayFromJsonNode(dependenciesObject, JSON::GetUtilityString(PackageDependencies)); if (packageDependencies) { for (auto& packageDependency : packageDependencies.value().get()) { - std::optional<std::string> id = JsonHelper::GetRawStringValueFromJsonNode(packageDependency, JsonHelper::GetUtilityString(PackageIdentifier)); + std::optional<std::string> id = JSON::GetRawStringValueFromJsonNode(packageDependency, JSON::GetUtilityString(PackageIdentifier)); if (id) { - Dependency pkg{ DependencyType::Package, std::move(id.value()) , JsonHelper::GetRawStringValueFromJsonNode(packageDependency, JsonHelper::GetUtilityString(MinimumVersion)).value_or("") }; + Dependency pkg{ DependencyType::Package, std::move(id.value()) , JSON::GetRawStringValueFromJsonNode(packageDependency, JSON::GetUtilityString(MinimumVersion)).value_or("") }; dependencyList.Add(std::move(pkg)); } } diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/SearchRequestSerializer_1_0.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/SearchRequestSerializer_1_0.cpp @@ -3,7 +3,7 @@ #include "pch.h" #include "Rest/Schema/IRestClient.h" #include "SearchRequestSerializer.h" -#include "Rest/Schema/JsonHelper.h" +#include <winget/JsonUtil.h> #include "Rest/Schema/CommonRestConstants.h" namespace AppInstaller::Repository::Rest::Schema::V1_0::Json @@ -62,12 +62,12 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json web::json::value json_body; if (searchRequest.MaximumResults > 0) { - json_body[JsonHelper::GetUtilityString(MaximumResults)] = searchRequest.MaximumResults; + json_body[JSON::GetUtilityString(MaximumResults)] = searchRequest.MaximumResults; } if (searchRequest.IsForEverything()) { - json_body[JsonHelper::GetUtilityString(FetchAllManifests)] = web::json::value::boolean(true); + json_body[JSON::GetUtilityString(FetchAllManifests)] = web::json::value::boolean(true); return json_body; } @@ -78,7 +78,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json std::optional<web::json::value> requestMatchJson = GetRequestMatchJsonObject(requestMatch); if (requestMatchJson) { - json_body[JsonHelper::GetUtilityString(Query)] = std::move(requestMatchJson.value()); + json_body[JSON::GetUtilityString(Query)] = std::move(requestMatchJson.value()); } } @@ -97,7 +97,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } } - json_body[JsonHelper::GetUtilityString(Filters)] = filters; + json_body[JSON::GetUtilityString(Filters)] = filters; } if (!searchRequest.Inclusions.empty()) @@ -115,7 +115,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json } } - json_body[JsonHelper::GetUtilityString(Inclusions)] = inclusions; + json_body[JSON::GetUtilityString(Inclusions)] = inclusions; } return json_body; @@ -143,7 +143,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json return {}; } - filter[JsonHelper::GetUtilityString(PackageMatchField)] = web::json::value::string(JsonHelper::GetUtilityString(matchField.value())); + filter[JSON::GetUtilityString(PackageMatchField)] = web::json::value::string(JSON::GetUtilityString(matchField.value())); std::optional<web::json::value> requestMatchJson = GetRequestMatchJsonObject(packageMatchFilter); if (!requestMatchJson) @@ -152,14 +152,14 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json return {}; } - filter[JsonHelper::GetUtilityString(RequestMatch)] = std::move(requestMatchJson.value()); + filter[JSON::GetUtilityString(RequestMatch)] = std::move(requestMatchJson.value()); return filter; } std::optional<web::json::value> SearchRequestSerializer::GetRequestMatchJsonObject(const AppInstaller::Repository::RequestMatch& requestMatch) const { web::json::value match = web::json::value::object(); - match[JsonHelper::GetUtilityString(KeyWord)] = web::json::value::string(JsonHelper::GetUtilityString(requestMatch.Value)); + match[JSON::GetUtilityString(KeyWord)] = web::json::value::string(JSON::GetUtilityString(requestMatch.Value)); std::optional<std::string_view> matchType = ConvertMatchTypeToString(requestMatch.Type); if (!matchType) @@ -168,7 +168,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json return {}; } - match[JsonHelper::GetUtilityString(MatchType)] = web::json::value::string(JsonHelper::GetUtilityString(matchType.value())); + match[JSON::GetUtilityString(MatchType)] = web::json::value::string(JSON::GetUtilityString(matchType.value())); return match; } diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/SearchResponseDeserializer_1_0.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/Json/SearchResponseDeserializer_1_0.cpp @@ -3,7 +3,7 @@ #include "pch.h" #include "Rest/Schema/IRestClient.h" #include "SearchResponseDeserializer.h" -#include "Rest/Schema/JsonHelper.h" +#include <winget/JsonUtil.h> #include "Rest/Schema/RestHelper.h" #include "Rest/Schema/CommonRestConstants.h" @@ -43,7 +43,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json IRestClient::SearchResult result; try { - std::optional<std::reference_wrapper<const web::json::array>> dataArray = JsonHelper::GetRawJsonArrayFromJsonNode(searchResponseObject, JsonHelper::GetUtilityString(Data)); + std::optional<std::reference_wrapper<const web::json::array>> dataArray = JSON::GetRawJsonArrayFromJsonNode(searchResponseObject, JSON::GetUtilityString(Data)); if (!dataArray || dataArray.value().get().size() == 0) { AICLI_LOG(Repo, Verbose, << "No search results returned."); @@ -52,33 +52,33 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0::Json for (auto& manifestItem : dataArray.value().get()) { - std::optional<std::string> packageId = JsonHelper::GetRawStringValueFromJsonNode(manifestItem, JsonHelper::GetUtilityString(PackageIdentifier)); - std::optional<std::string> packageName = JsonHelper::GetRawStringValueFromJsonNode(manifestItem, JsonHelper::GetUtilityString(PackageName)); - std::optional<std::string> publisher = JsonHelper::GetRawStringValueFromJsonNode(manifestItem, JsonHelper::GetUtilityString(Publisher)); + std::optional<std::string> packageId = JSON::GetRawStringValueFromJsonNode(manifestItem, JSON::GetUtilityString(PackageIdentifier)); + std::optional<std::string> packageName = JSON::GetRawStringValueFromJsonNode(manifestItem, JSON::GetUtilityString(PackageName)); + std::optional<std::string> publisher = JSON::GetRawStringValueFromJsonNode(manifestItem, JSON::GetUtilityString(Publisher)); - if (!JsonHelper::IsValidNonEmptyStringValue(packageId) || !JsonHelper::IsValidNonEmptyStringValue(packageName) || !JsonHelper::IsValidNonEmptyStringValue(publisher)) + if (!JSON::IsValidNonEmptyStringValue(packageId) || !JSON::IsValidNonEmptyStringValue(packageName) || !JSON::IsValidNonEmptyStringValue(publisher)) { AICLI_LOG(Repo, Error, << "Missing required package fields in manifest search results."); return {}; } - std::optional<std::reference_wrapper<const web::json::array>> versionValue = JsonHelper::GetRawJsonArrayFromJsonNode(manifestItem, JsonHelper::GetUtilityString(Versions)); + std::optional<std::reference_wrapper<const web::json::array>> versionValue = JSON::GetRawJsonArrayFromJsonNode(manifestItem, JSON::GetUtilityString(Versions)); std::vector<IRestClient::VersionInfo> versionList; if (versionValue) { for (auto& versionItem : versionValue.value().get()) { - std::optional<std::string> version = JsonHelper::GetRawStringValueFromJsonNode(versionItem, JsonHelper::GetUtilityString(PackageVersion)); - if (!JsonHelper::IsValidNonEmptyStringValue(version)) + std::optional<std::string> version = JSON::GetRawStringValueFromJsonNode(versionItem, JSON::GetUtilityString(PackageVersion)); + if (!JSON::IsValidNonEmptyStringValue(version)) { AICLI_LOG(Repo, Error, << "Received incomplete package version in package: " << packageId.value()); return {}; } - std::string channel = JsonHelper::GetRawStringValueFromJsonNode(versionItem, JsonHelper::GetUtilityString(Channel)).value_or(""); - std::vector<std::string> packageFamilyNames = RestHelper::GetUniqueItems(JsonHelper::GetRawStringArrayFromJsonNode(versionItem, JsonHelper::GetUtilityString(PackageFamilyNames))); - std::vector<std::string> productCodes = RestHelper::GetUniqueItems(JsonHelper::GetRawStringArrayFromJsonNode(versionItem, JsonHelper::GetUtilityString(ProductCodes))); + std::string channel = JSON::GetRawStringValueFromJsonNode(versionItem, JSON::GetUtilityString(Channel)).value_or(""); + std::vector<std::string> packageFamilyNames = RestHelper::GetUniqueItems(JSON::GetRawStringArrayFromJsonNode(versionItem, JSON::GetUtilityString(PackageFamilyNames))); + std::vector<std::string> productCodes = RestHelper::GetUniqueItems(JSON::GetRawStringArrayFromJsonNode(versionItem, JSON::GetUtilityString(ProductCodes))); versionList.emplace_back(IRestClient::VersionInfo{ AppInstaller::Utility::VersionAndChannel{std::move(version.value()), std::move(channel)}, {}, std::move(packageFamilyNames), std::move(productCodes)}); diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_0/RestInterface_1_0.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_0/RestInterface_1_0.cpp @@ -4,8 +4,8 @@ #include "Rest/Schema/1_0/Interface.h" #include "Rest/Schema/IRestClient.h" #include "Rest/Schema/HttpClientHelper.h" -#include "Rest/Schema/JsonHelper.h" -#include "winget/ManifestValidation.h" +#include <winget/JsonUtil.h> +#include <winget/ManifestValidation.h> #include "Rest/Schema/RestHelper.h" #include "Rest/Schema/CommonRestConstants.h" #include "Rest/Schema/1_0/Json/ManifestDeserializer.h" @@ -25,16 +25,16 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0 utility::string_t GetSearchEndpoint(const std::string& restApiUri) { - return RestHelper::AppendPathToUri(JsonHelper::GetUtilityString(restApiUri), JsonHelper::GetUtilityString(ManifestSearchPostEndpoint)); + return RestHelper::AppendPathToUri(JSON::GetUtilityString(restApiUri), JSON::GetUtilityString(ManifestSearchPostEndpoint)); } utility::string_t GetManifestByVersionEndpoint( const std::string& restApiUri, const std::string& packageId, const std::map<std::string_view, std::string>& queryParameters) { utility::string_t getManifestEndpoint = RestHelper::AppendPathToUri( - JsonHelper::GetUtilityString(restApiUri), JsonHelper::GetUtilityString(ManifestByVersionAndChannelGetEndpoint)); + JSON::GetUtilityString(restApiUri), JSON::GetUtilityString(ManifestByVersionAndChannelGetEndpoint)); - utility::string_t getManifestWithPackageIdPath = RestHelper::AppendPathToUri(getManifestEndpoint, JsonHelper::GetUtilityString(packageId)); + utility::string_t getManifestWithPackageIdPath = RestHelper::AppendPathToUri(getManifestEndpoint, JSON::GetUtilityString(packageId)); // Create the endpoint with query parameters return RestHelper::AppendQueryParamsToUri(getManifestWithPackageIdPath, queryParameters); @@ -43,10 +43,10 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0 Interface::Interface(const std::string& restApi, const HttpClientHelper& httpClientHelper) : m_restApiUri(restApi), m_httpClientHelper(httpClientHelper) { - THROW_HR_IF(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_URL, !RestHelper::IsValidUri(JsonHelper::GetUtilityString(restApi))); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_RESTSOURCE_INVALID_URL, !RestHelper::IsValidUri(JSON::GetUtilityString(restApi))); m_searchEndpoint = GetSearchEndpoint(m_restApiUri); - m_requiredRestApiHeaders.emplace(JsonHelper::GetUtilityString(ContractVersion), JsonHelper::GetUtilityString(Version_1_0_0.ToString())); + m_requiredRestApiHeaders.emplace(JSON::GetUtilityString(ContractVersion), JSON::GetUtilityString(Version_1_0_0.ToString())); } Utility::Version Interface::GetVersion() const @@ -80,7 +80,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_0 if (!continuationToken.empty()) { AICLI_LOG(Repo, Verbose, << "Received continuation token. Retrieving more results."); - searchHeaders.insert_or_assign(JsonHelper::GetUtilityString(ContinuationToken), continuationToken); + searchHeaders.insert_or_assign(JSON::GetUtilityString(ContinuationToken), continuationToken); } std::optional<web::json::value> jsonObject = m_httpClientHelper.HandlePost(m_searchEndpoint, GetValidatedSearchBody(request), searchHeaders); diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer.h b/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer.h @@ -8,11 +8,13 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json // Manifest Deserializer. struct ManifestDeserializer : public V1_0::Json::ManifestDeserializer { - protected: - std::optional<Manifest::ManifestInstaller> DeserializeInstaller(const web::json::value& installerJsonObject) const override; + std::vector<Manifest::AppsAndFeaturesEntry> DeserializeAppsAndFeaturesEntries(const web::json::array& entries) const override; std::optional<Manifest::ManifestLocalization> DeserializeLocale(const web::json::value& localeJsonObject) const override; + protected: + std::optional<Manifest::ManifestInstaller> DeserializeInstaller(const web::json::value& installerJsonObject) const override; + Manifest::InstallerTypeEnum ConvertToInstallerType(std::string_view in) const override; }; } diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer_1_1.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_1/Json/ManifestDeserializer_1_1.cpp @@ -2,7 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include "ManifestDeserializer.h" -#include "Rest/Schema/JsonHelper.h" +#include <winget/JsonUtil.h> using namespace AppInstaller::Manifest; @@ -41,6 +41,31 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json constexpr std::string_view AgreementUrl = "AgreementUrl"sv; } + std::vector<Manifest::AppsAndFeaturesEntry> ManifestDeserializer::DeserializeAppsAndFeaturesEntries(const web::json::array& entries) const + { + std::vector<Manifest::AppsAndFeaturesEntry> result; + + for (auto& arpEntryNode : entries) + { + AppsAndFeaturesEntry arpEntry; + arpEntry.DisplayName = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(DisplayName)).value_or(""); + arpEntry.Publisher = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(Publisher)).value_or(""); + arpEntry.DisplayVersion = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(DisplayVersion)).value_or(""); + arpEntry.ProductCode = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(ProductCode)).value_or(""); + arpEntry.UpgradeCode = JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(UpgradeCode)).value_or(""); + arpEntry.InstallerType = Manifest::ConvertToInstallerTypeEnum(JSON::GetRawStringValueFromJsonNode(arpEntryNode, JSON::GetUtilityString(InstallerType)).value_or("")); + + // Only add when at least one field is valid + if (!arpEntry.DisplayName.empty() || !arpEntry.Publisher.empty() || !arpEntry.DisplayVersion.empty() || + !arpEntry.ProductCode.empty() || !arpEntry.UpgradeCode.empty() || arpEntry.InstallerType != InstallerTypeEnum::Unknown) + { + result.emplace_back(std::move(arpEntry)); + } + } + + return result; + } + Manifest::InstallerTypeEnum ManifestDeserializer::ConvertToInstallerType(std::string_view in) const { std::string inStrLower = Utility::ToLower(in); @@ -61,22 +86,22 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json { auto& installer = result.value(); - installer.ProductId = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(MSStoreProductIdentifier)).value_or(""); - installer.ReleaseDate = JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(ReleaseDate)).value_or(""); - installer.InstallerAbortsTerminal = JsonHelper::GetRawBoolValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallerAbortsTerminal)).value_or(false); - installer.InstallLocationRequired = JsonHelper::GetRawBoolValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(InstallLocationRequired)).value_or(false); - installer.RequireExplicitUpgrade = JsonHelper::GetRawBoolValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(RequireExplicitUpgrade)).value_or(false); + installer.ProductId = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(MSStoreProductIdentifier)).value_or(""); + installer.ReleaseDate = JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(ReleaseDate)).value_or(""); + installer.InstallerAbortsTerminal = JSON::GetRawBoolValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallerAbortsTerminal)).value_or(false); + installer.InstallLocationRequired = JSON::GetRawBoolValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(InstallLocationRequired)).value_or(false); + installer.RequireExplicitUpgrade = JSON::GetRawBoolValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(RequireExplicitUpgrade)).value_or(false); installer.ElevationRequirement = Manifest::ConvertToElevationRequirementEnum( - JsonHelper::GetRawStringValueFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(ElevationRequirement)).value_or("")); + JSON::GetRawStringValueFromJsonNode(installerJsonObject, JSON::GetUtilityString(ElevationRequirement)).value_or("")); // list of unsupported OS architectures - std::optional<std::reference_wrapper<const web::json::array>> unsupportedOSArchitectures = JsonHelper::GetRawJsonArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(UnsupportedOSArchitectures)); + std::optional<std::reference_wrapper<const web::json::array>> unsupportedOSArchitectures = JSON::GetRawJsonArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(UnsupportedOSArchitectures)); if (unsupportedOSArchitectures) { for (auto& archValue : unsupportedOSArchitectures.value().get()) { - std::optional<std::string> arch = JsonHelper::GetRawStringValueFromJsonValue(archValue); - if (JsonHelper::IsValidNonEmptyStringValue(arch)) + std::optional<std::string> arch = JSON::GetRawStringValueFromJsonValue(archValue); + if (JSON::IsValidNonEmptyStringValue(arch)) { auto archEnum = Utility::ConvertToArchitectureEnum(arch.value()); @@ -95,46 +120,30 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json } // Apps and Features Entries - std::optional<std::reference_wrapper<const web::json::array>> arpEntriesNode = JsonHelper::GetRawJsonArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(AppsAndFeaturesEntries)); + std::optional<std::reference_wrapper<const web::json::array>> arpEntriesNode = JSON::GetRawJsonArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(AppsAndFeaturesEntries)); if (arpEntriesNode) { - for (auto& arpEntryNode : arpEntriesNode.value().get()) - { - AppsAndFeaturesEntry arpEntry; - arpEntry.DisplayName = JsonHelper::GetRawStringValueFromJsonNode(arpEntryNode, JsonHelper::GetUtilityString(DisplayName)).value_or(""); - arpEntry.Publisher = JsonHelper::GetRawStringValueFromJsonNode(arpEntryNode, JsonHelper::GetUtilityString(Publisher)).value_or(""); - arpEntry.DisplayVersion = JsonHelper::GetRawStringValueFromJsonNode(arpEntryNode, JsonHelper::GetUtilityString(DisplayVersion)).value_or(""); - arpEntry.ProductCode = JsonHelper::GetRawStringValueFromJsonNode(arpEntryNode, JsonHelper::GetUtilityString(ProductCode)).value_or(""); - arpEntry.UpgradeCode = JsonHelper::GetRawStringValueFromJsonNode(arpEntryNode, JsonHelper::GetUtilityString(UpgradeCode)).value_or(""); - arpEntry.InstallerType = Manifest::ConvertToInstallerTypeEnum(JsonHelper::GetRawStringValueFromJsonNode(arpEntryNode, JsonHelper::GetUtilityString(InstallerType)).value_or("")); - - // Only add when at least one field is valid - if (!arpEntry.DisplayName.empty() || !arpEntry.Publisher.empty() || !arpEntry.DisplayVersion.empty() || - !arpEntry.ProductCode.empty() || !arpEntry.UpgradeCode.empty() || arpEntry.InstallerType != InstallerTypeEnum::Unknown) - { - installer.AppsAndFeaturesEntries.emplace_back(std::move(arpEntry)); - } - } + installer.AppsAndFeaturesEntries = DeserializeAppsAndFeaturesEntries(arpEntriesNode.value()); } // Markets - std::optional<std::reference_wrapper<const web::json::value>> marketsNode = JsonHelper::GetJsonValueFromNode(installerJsonObject, JsonHelper::GetUtilityString(Markets)); + std::optional<std::reference_wrapper<const web::json::value>> marketsNode = JSON::GetJsonValueFromNode(installerJsonObject, JSON::GetUtilityString(Markets)); if (marketsNode && !marketsNode.value().get().is_null()) { installer.Markets.ExcludedMarkets = V1_0::Json::ManifestDeserializer::ConvertToManifestStringArray( - JsonHelper::GetRawStringArrayFromJsonNode(marketsNode.value().get(), JsonHelper::GetUtilityString(ExcludedMarkets))); + JSON::GetRawStringArrayFromJsonNode(marketsNode.value().get(), JSON::GetUtilityString(ExcludedMarkets))); installer.Markets.AllowedMarkets = V1_0::Json::ManifestDeserializer::ConvertToManifestStringArray( - JsonHelper::GetRawStringArrayFromJsonNode(marketsNode.value().get(), JsonHelper::GetUtilityString(AllowedMarkets))); + JSON::GetRawStringArrayFromJsonNode(marketsNode.value().get(), JSON::GetUtilityString(AllowedMarkets))); } // Expected return codes - std::optional<std::reference_wrapper<const web::json::array>> expectedReturnCodesNode = JsonHelper::GetRawJsonArrayFromJsonNode(installerJsonObject, JsonHelper::GetUtilityString(ExpectedReturnCodes)); + std::optional<std::reference_wrapper<const web::json::array>> expectedReturnCodesNode = JSON::GetRawJsonArrayFromJsonNode(installerJsonObject, JSON::GetUtilityString(ExpectedReturnCodes)); if (expectedReturnCodesNode) { for (auto& returnCodeNode : expectedReturnCodesNode.value().get()) { - ExpectedReturnCodeEnum returnResponse = Manifest::ConvertToExpectedReturnCodeEnum(JsonHelper::GetRawStringValueFromJsonNode(returnCodeNode, JsonHelper::GetUtilityString(ReturnResponse)).value_or("")); - DWORD installerReturnCode = static_cast<DWORD>(JsonHelper::GetRawIntValueFromJsonNode(returnCodeNode, JsonHelper::GetUtilityString(InstallerReturnCode)).value_or(0)); + ExpectedReturnCodeEnum returnResponse = Manifest::ConvertToExpectedReturnCodeEnum(JSON::GetRawStringValueFromJsonNode(returnCodeNode, JSON::GetUtilityString(ReturnResponse)).value_or("")); + DWORD installerReturnCode = static_cast<DWORD>(JSON::GetRawIntValueFromJsonNode(returnCodeNode, JSON::GetUtilityString(InstallerReturnCode)).value_or(0)); // Only add when it is valid if (installerReturnCode != 0 && returnResponse != ExpectedReturnCodeEnum::Unknown) @@ -175,7 +184,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json TryParseStringLocaleField<Manifest::Localization::ReleaseNotesUrl>(locale, localeJsonObject, ReleaseNotesUrl); // Agreements - auto agreementsNode = JsonHelper::GetRawJsonArrayFromJsonNode(localeJsonObject, JsonHelper::GetUtilityString(Agreements)); + auto agreementsNode = JSON::GetRawJsonArrayFromJsonNode(localeJsonObject, JSON::GetUtilityString(Agreements)); if (agreementsNode) { std::vector<Manifest::Agreement> agreements; @@ -183,9 +192,9 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1::Json { Manifest::Agreement agreementEntry; - agreementEntry.Label = JsonHelper::GetRawStringValueFromJsonNode(agreementNode, JsonHelper::GetUtilityString(AgreementLabel)).value_or(""); - agreementEntry.AgreementText = JsonHelper::GetRawStringValueFromJsonNode(agreementNode, JsonHelper::GetUtilityString(Agreement)).value_or(""); - agreementEntry.AgreementUrl = JsonHelper::GetRawStringValueFromJsonNode(agreementNode, JsonHelper::GetUtilityString(AgreementUrl)).value_or(""); + agreementEntry.Label = JSON::GetRawStringValueFromJsonNode(agreementNode, JSON::GetUtilityString(AgreementLabel)).value_or(""); + agreementEntry.AgreementText = JSON::GetRawStringValueFromJsonNode(agreementNode, JSON::GetUtilityString(Agreement)).value_or(""); + agreementEntry.AgreementUrl = JSON::GetRawStringValueFromJsonNode(agreementNode, JSON::GetUtilityString(AgreementUrl)).value_or(""); if (!agreementEntry.Label.empty() || !agreementEntry.AgreementText.empty() || !agreementEntry.AgreementUrl.empty()) { diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/1_1/RestInterface_1_1.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/1_1/RestInterface_1_1.cpp @@ -4,7 +4,7 @@ #include "Rest/Schema/1_1/Interface.h" #include "Rest/Schema/IRestClient.h" #include "Rest/Schema/HttpClientHelper.h" -#include "Rest/Schema/JsonHelper.h" +#include <winget/JsonUtil.h> #include "Rest/Schema/RestHelper.h" #include "Rest/Schema/CommonRestConstants.h" #include "Rest/Schema/1_1/Json/ManifestDeserializer.h" @@ -33,7 +33,7 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1 const std::unordered_map<utility::string_t, utility::string_t>& additionalHeaders, const HttpClientHelper& httpClientHelper) : V1_0::Interface(restApi, httpClientHelper), m_information(std::move(information)) { - m_requiredRestApiHeaders[JsonHelper::GetUtilityString(ContractVersion)] = JsonHelper::GetUtilityString(Version_1_1_0.ToString()); + m_requiredRestApiHeaders[JSON::GetUtilityString(ContractVersion)] = JSON::GetUtilityString(Version_1_1_0.ToString()); if (!additionalHeaders.empty()) { @@ -140,8 +140,8 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1 if (result.Matches.size() == 0) { - auto requiredPackageMatchFields = JsonHelper::GetRawStringArrayFromJsonNode(searchResponseObject, JsonHelper::GetUtilityString(RequiredPackageMatchFields)); - auto unsupportedPackageMatchFields = JsonHelper::GetRawStringArrayFromJsonNode(searchResponseObject, JsonHelper::GetUtilityString(UnsupportedPackageMatchFields)); + auto requiredPackageMatchFields = JSON::GetRawStringArrayFromJsonNode(searchResponseObject, JSON::GetUtilityString(RequiredPackageMatchFields)); + auto unsupportedPackageMatchFields = JSON::GetRawStringArrayFromJsonNode(searchResponseObject, JSON::GetUtilityString(UnsupportedPackageMatchFields)); if (requiredPackageMatchFields.size() != 0 || unsupportedPackageMatchFields.size() != 0) { @@ -160,8 +160,8 @@ namespace AppInstaller::Repository::Rest::Schema::V1_1 if (result.size() == 0) { - auto requiredQueryParameters = JsonHelper::GetRawStringArrayFromJsonNode(manifestsResponseObject, JsonHelper::GetUtilityString(RequiredQueryParameters)); - auto unsupportedQueryParameters = JsonHelper::GetRawStringArrayFromJsonNode(manifestsResponseObject, JsonHelper::GetUtilityString(UnsupportedQueryParameters)); + auto requiredQueryParameters = JSON::GetRawStringArrayFromJsonNode(manifestsResponseObject, JSON::GetUtilityString(RequiredQueryParameters)); + auto unsupportedQueryParameters = JSON::GetRawStringArrayFromJsonNode(manifestsResponseObject, JSON::GetUtilityString(UnsupportedQueryParameters)); if (requiredQueryParameters.size() != 0 || unsupportedQueryParameters.size() != 0) { diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/InformationResponseDeserializer.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/InformationResponseDeserializer.cpp @@ -2,7 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include "Rest/Schema/IRestClient.h" -#include "Rest/Schema/JsonHelper.h" +#include <winget/JsonUtil.h> #include "Rest/Schema/CommonRestConstants.h" #include "InformationResponseDeserializer.h" @@ -47,7 +47,7 @@ namespace AppInstaller::Repository::Rest::Schema return {}; } - std::optional<std::reference_wrapper<const web::json::value>> data = JsonHelper::GetJsonValueFromNode(dataObject, JsonHelper::GetUtilityString(Data)); + std::optional<std::reference_wrapper<const web::json::value>> data = JSON::GetJsonValueFromNode(dataObject, JSON::GetUtilityString(Data)); if (!data) { AICLI_LOG(Repo, Error, << "Missing data"); @@ -55,14 +55,14 @@ namespace AppInstaller::Repository::Rest::Schema } const auto& dataValue = data.value().get(); - std::optional<std::string> sourceId = JsonHelper::GetRawStringValueFromJsonNode(dataValue, JsonHelper::GetUtilityString(SourceIdentifier)); - if (!JsonHelper::IsValidNonEmptyStringValue(sourceId)) + std::optional<std::string> sourceId = JSON::GetRawStringValueFromJsonNode(dataValue, JSON::GetUtilityString(SourceIdentifier)); + if (!JSON::IsValidNonEmptyStringValue(sourceId)) { AICLI_LOG(Repo, Error, << "Missing source identifier"); return {}; } - std::vector<std::string> allVersions = JsonHelper::GetRawStringArrayFromJsonNode(dataValue, JsonHelper::GetUtilityString(ServerSupportedVersions)); + std::vector<std::string> allVersions = JSON::GetRawStringArrayFromJsonNode(dataValue, JSON::GetUtilityString(ServerSupportedVersions)); if (allVersions.size() == 0) { AICLI_LOG(Repo, Error, << "Missing supported versions."); @@ -71,13 +71,13 @@ namespace AppInstaller::Repository::Rest::Schema IRestClient::Information info{ std::move(sourceId.value()), std::move(allVersions) }; - auto agreements = JsonHelper::GetJsonValueFromNode(dataValue, JsonHelper::GetUtilityString(SourceAgreements)); + auto agreements = JSON::GetJsonValueFromNode(dataValue, JSON::GetUtilityString(SourceAgreements)); if (agreements) { const auto& agreementsValue = agreements.value().get(); - auto agreementsIdentifier = JsonHelper::GetRawStringValueFromJsonNode(agreementsValue, JsonHelper::GetUtilityString(SourceAgreementsIdentifier)); - if (!JsonHelper::IsValidNonEmptyStringValue(agreementsIdentifier)) + auto agreementsIdentifier = JSON::GetRawStringValueFromJsonNode(agreementsValue, JSON::GetUtilityString(SourceAgreementsIdentifier)); + if (!JSON::IsValidNonEmptyStringValue(agreementsIdentifier)) { AICLI_LOG(Repo, Error, << "SourceAgreements node exists but AgreementsIdentifier is missing."); return {}; @@ -85,27 +85,27 @@ namespace AppInstaller::Repository::Rest::Schema info.SourceAgreementsIdentifier = std::move(agreementsIdentifier.value()); - auto agreementsContent = JsonHelper::GetRawJsonArrayFromJsonNode(agreementsValue, JsonHelper::GetUtilityString(SourceAgreementsContent)); + auto agreementsContent = JSON::GetRawJsonArrayFromJsonNode(agreementsValue, JSON::GetUtilityString(SourceAgreementsContent)); if (agreementsContent) { for (auto const& agreementNode : agreementsContent.value().get()) { IRestClient::SourceAgreementEntry agreementEntry; - std::optional<std::string> label = JsonHelper::GetRawStringValueFromJsonNode(agreementNode, JsonHelper::GetUtilityString(SourceAgreementLabel)); - if (JsonHelper::IsValidNonEmptyStringValue(label)) + std::optional<std::string> label = JSON::GetRawStringValueFromJsonNode(agreementNode, JSON::GetUtilityString(SourceAgreementLabel)); + if (JSON::IsValidNonEmptyStringValue(label)) { agreementEntry.Label = std::move(label.value()); } - std::optional<std::string> text = JsonHelper::GetRawStringValueFromJsonNode(agreementNode, JsonHelper::GetUtilityString(SourceAgreementText)); - if (JsonHelper::IsValidNonEmptyStringValue(text)) + std::optional<std::string> text = JSON::GetRawStringValueFromJsonNode(agreementNode, JSON::GetUtilityString(SourceAgreementText)); + if (JSON::IsValidNonEmptyStringValue(text)) { agreementEntry.Text = std::move(text.value()); } - std::optional<std::string> url = JsonHelper::GetRawStringValueFromJsonNode(agreementNode, JsonHelper::GetUtilityString(SourceAgreementUrl)); - if (JsonHelper::IsValidNonEmptyStringValue(url)) + std::optional<std::string> url = JSON::GetRawStringValueFromJsonNode(agreementNode, JSON::GetUtilityString(SourceAgreementUrl)); + if (JSON::IsValidNonEmptyStringValue(url)) { agreementEntry.Url = std::move(url.value()); } @@ -118,10 +118,10 @@ namespace AppInstaller::Repository::Rest::Schema } } - info.RequiredPackageMatchFields = JsonHelper::GetRawStringArrayFromJsonNode(dataValue, JsonHelper::GetUtilityString(RequiredPackageMatchFields)); - info.UnsupportedPackageMatchFields = JsonHelper::GetRawStringArrayFromJsonNode(dataValue, JsonHelper::GetUtilityString(UnsupportedPackageMatchFields)); - info.RequiredQueryParameters = JsonHelper::GetRawStringArrayFromJsonNode(dataValue, JsonHelper::GetUtilityString(RequiredQueryParameters)); - info.UnsupportedQueryParameters = JsonHelper::GetRawStringArrayFromJsonNode(dataValue, JsonHelper::GetUtilityString(UnsupportedQueryParameters)); + info.RequiredPackageMatchFields = JSON::GetRawStringArrayFromJsonNode(dataValue, JSON::GetUtilityString(RequiredPackageMatchFields)); + info.UnsupportedPackageMatchFields = JSON::GetRawStringArrayFromJsonNode(dataValue, JSON::GetUtilityString(UnsupportedPackageMatchFields)); + info.RequiredQueryParameters = JSON::GetRawStringArrayFromJsonNode(dataValue, JSON::GetUtilityString(RequiredQueryParameters)); + info.UnsupportedQueryParameters = JSON::GetRawStringArrayFromJsonNode(dataValue, JSON::GetUtilityString(UnsupportedQueryParameters)); return info; } diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/JsonHelper.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/JsonHelper.cpp @@ -1,133 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -#include "pch.h" -#include "JsonHelper.h" - -namespace AppInstaller::Repository::Rest::Schema -{ - utility::string_t JsonHelper::GetUtilityString(std::string_view nodeName) - { - return utility::conversions::to_string_t(nodeName.data()); - } - - std::optional<std::reference_wrapper<const web::json::value>> JsonHelper::GetJsonValueFromNode(const web::json::value& node, const utility::string_t& keyName) - { - if (node.is_null() || !node.has_field(keyName)) - { - return {}; - } - - return node.at(keyName); - } - - std::optional<std::string> JsonHelper::GetRawStringValueFromJsonValue(const web::json::value& value) - { - if (value.is_null() || !value.is_string()) - { - return {}; - } - - return utility::conversions::to_utf8string(value.as_string()); - } - - std::optional<std::string> JsonHelper::GetRawStringValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName) - { - std::optional<std::reference_wrapper<const web::json::value>> jsonValue = GetJsonValueFromNode(node, keyName); - - if (jsonValue) - { - return GetRawStringValueFromJsonValue(jsonValue.value().get()); - } - - return {}; - } - - std::optional<int> JsonHelper::GetRawIntValueFromJsonValue(const web::json::value& value) - { - if (value.is_null() || !value.is_integer()) - { - return {}; - } - - return value.as_integer(); - } - - std::optional<int> JsonHelper::GetRawIntValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName) - { - std::optional<std::reference_wrapper<const web::json::value>> jsonValue = GetJsonValueFromNode(node, keyName); - - if (jsonValue) - { - return GetRawIntValueFromJsonValue(jsonValue.value().get()); - } - - return {}; - } - - std::optional<bool> JsonHelper::GetRawBoolValueFromJsonValue(const web::json::value& value) - { - if (value.is_null() || !value.is_boolean()) - { - return {}; - } - - return value.as_bool(); - } - - std::optional<bool> JsonHelper::GetRawBoolValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName) - { - std::optional<std::reference_wrapper<const web::json::value>> jsonValue = GetJsonValueFromNode(node, keyName); - - if (jsonValue) - { - return GetRawBoolValueFromJsonValue(jsonValue.value().get()); - } - - return {}; - } - - std::optional<std::reference_wrapper<const web::json::array>> JsonHelper::GetRawJsonArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName) - { - std::optional<std::reference_wrapper<const web::json::value>> jsonValue = GetJsonValueFromNode(node, keyName); - - if (!jsonValue || !jsonValue.value().get().is_array()) - { - return {}; - } - - return jsonValue.value().get().as_array(); - } - - std::vector<std::string> JsonHelper::GetRawStringArrayFromJsonNode( - const web::json::value& node, const utility::string_t& keyName) - { - std::optional<std::reference_wrapper<const web::json::array>> arrayValue = GetRawJsonArrayFromJsonNode(node, keyName); - - std::vector<std::string> result; - if (!arrayValue) - { - return result; - } - - for (auto& value : arrayValue.value().get()) - { - std::optional<std::string> item = JsonHelper::GetRawStringValueFromJsonValue(value); - if (item) - { - result.emplace_back(std::move(item.value())); - } - } - - return result; - } - - bool JsonHelper::IsValidNonEmptyStringValue(std::optional<std::string>& value) - { - if (Utility::IsEmptyOrWhitespace(value.value_or(""))) - { - return false; - } - - return true; - } -} diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/JsonHelper.h b/src/AppInstallerRepositoryCore/Rest/Schema/JsonHelper.h @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -#pragma once -#include <cpprest/json.h> -#include "winget/Manifest.h" - -namespace AppInstaller::Repository::Rest::Schema -{ - // Json helper. - struct JsonHelper - { - static std::optional<std::reference_wrapper<const web::json::value>> GetJsonValueFromNode(const web::json::value& node, const utility::string_t& keyName); - - static std::optional<std::string> GetRawStringValueFromJsonValue(const web::json::value& value); - - static std::optional<std::string> GetRawStringValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName); - - static std::optional<bool> GetRawBoolValueFromJsonValue(const web::json::value& value); - - static std::optional<bool> GetRawBoolValueFromJsonNode(const web::json::value& node, const utility::string_t& keyName); - - static std::optional<std::reference_wrapper<const web::json::array>> GetRawJsonArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName); - - static std::optional<int> GetRawIntValueFromJsonValue(const web::json::value& value); - - static std::optional<int> GetRawIntValueFromJsonNode(const web::json::value& value, const utility::string_t& keyName); - - static utility::string_t GetUtilityString(std::string_view nodeName); - - static std::vector<std::string> GetRawStringArrayFromJsonNode(const web::json::value& node, const utility::string_t& keyName); - - static bool IsValidNonEmptyStringValue(std::optional<std::string>& value); - }; -} diff --git a/src/AppInstallerRepositoryCore/Rest/Schema/RestHelper.cpp b/src/AppInstallerRepositoryCore/Rest/Schema/RestHelper.cpp @@ -2,7 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include "RestHelper.h" -#include "Rest/Schema/JsonHelper.h" +#include <winget/JsonUtil.h> #include "Rest/Schema/CommonRestConstants.h" using namespace AppInstaller::Repository::Rest::Schema; @@ -25,7 +25,7 @@ namespace AppInstaller::Repository::Rest::Schema } // Encode the Uri - return web::uri::encode_uri(JsonHelper::GetUtilityString(uri)); + return web::uri::encode_uri(JSON::GetUtilityString(uri)); } bool RestHelper::IsValidUri(const utility::string_t& restApiUri) @@ -62,7 +62,7 @@ namespace AppInstaller::Repository::Rest::Schema std::optional<utility::string_t> RestHelper::GetContinuationToken(const web::json::value& jsonObject) { - std::optional<std::string> continuationToken = JsonHelper::GetRawStringValueFromJsonNode(jsonObject, JsonHelper::GetUtilityString(ContinuationToken)); + std::optional<std::string> continuationToken = JSON::GetRawStringValueFromJsonNode(jsonObject, JSON::GetUtilityString(ContinuationToken)); if (continuationToken) { diff --git a/src/AppInstallerRepositoryCore/pch.h b/src/AppInstallerRepositoryCore/pch.h @@ -59,6 +59,8 @@ #include <type_traits> #include <utility> +#include <json/json.h> + #pragma warning( push ) #pragma warning ( disable : 26495 26439 ) #include <cpprest/http_client.h> diff --git a/src/WinGetUtil/Exports.cpp b/src/WinGetUtil/Exports.cpp @@ -11,14 +11,24 @@ #include <Microsoft/SQLiteIndex.h> #include <winget/ManifestYamlParser.h> #include <winget/ThreadGlobals.h> +#include <winget/InstallerMetadataCollectionContext.h> #include <PackageDependenciesValidation.h> #include <ArpVersionValidation.h> using namespace AppInstaller::Utility; using namespace AppInstaller::Manifest; using namespace AppInstaller::Repository; +using namespace AppInstaller::Repository::Metadata; using namespace AppInstaller::Repository::Microsoft; +namespace +{ + std::filesystem::path GetPathOrEmpty(WINGET_STRING potentiallyNullPath) + { + return potentiallyNullPath ? std::filesystem::path{ potentiallyNullPath } : std::filesystem::path{}; + } +} + extern "C" { WINGET_UTIL_API WinGetLoggingInit(WINGET_STRING logPath) try @@ -144,7 +154,8 @@ extern "C" WINGET_UTIL_API WinGetSQLiteIndexRemoveManifest( WINGET_SQLITE_INDEX_HANDLE index, - WINGET_STRING manifestPath, WINGET_STRING relativePath) try + WINGET_STRING manifestPath, + WINGET_STRING relativePath) try { THROW_HR_IF(E_INVALIDARG, !index); THROW_HR_IF(E_INVALIDARG, !manifestPath); @@ -487,4 +498,78 @@ extern "C" return S_OK; } CATCH_RETURN() + + WINGET_UTIL_API WinGetBeginInstallerMetadataCollection( + WINGET_STRING inputJSON, + WINGET_STRING logFilePath, + WinGetBeginInstallerMetadataCollectionOptions options, + WINGET_INSTALLER_METADATA_COLLECTION_HANDLE* collectionHandle) try + { + THROW_HR_IF(E_INVALIDARG, !inputJSON); + THROW_HR_IF(E_INVALIDARG, !collectionHandle); + THROW_HR_IF(E_INVALIDARG, !!*collectionHandle); + // Flags specifying what inputJSON means are mutually exclusive + THROW_HR_IF(E_INVALIDARG, !WI_IsClearOrSingleFlagSetInMask(options, + WinGetBeginInstallerMetadataCollectionOption_InputIsFilePath | WinGetBeginInstallerMetadataCollectionOption_InputIsURI)); + + std::unique_ptr<InstallerMetadataCollectionContext> result; + + if (WI_IsFlagSet(options, WinGetBeginInstallerMetadataCollectionOption_InputIsFilePath)) + { + result = InstallerMetadataCollectionContext::FromFile(inputJSON, GetPathOrEmpty(logFilePath)); + } + else if (WI_IsFlagSet(options, WinGetBeginInstallerMetadataCollectionOption_InputIsURI)) + { + result = InstallerMetadataCollectionContext::FromURI(inputJSON, GetPathOrEmpty(logFilePath)); + } + else + { + result = InstallerMetadataCollectionContext::FromJSON(inputJSON, GetPathOrEmpty(logFilePath)); + } + + *collectionHandle = static_cast<WINGET_INSTALLER_METADATA_COLLECTION_HANDLE>(result.release()); + + return S_OK; + } + CATCH_RETURN() + + WINGET_UTIL_API WinGetCompleteInstallerMetadataCollection( + WINGET_INSTALLER_METADATA_COLLECTION_HANDLE collectionHandle, + WINGET_STRING outputFilePath, + WinGetCompleteInstallerMetadataCollectionOptions options) try + { + THROW_HR_IF(E_INVALIDARG, !collectionHandle); + + // Since we always free the handle from calling this function, we can just store it in a unique_ptr from the start + std::unique_ptr<InstallerMetadataCollectionContext> context{ reinterpret_cast<InstallerMetadataCollectionContext*>(collectionHandle) }; + + if (WI_IsFlagSet(options, WinGetCompleteInstallerMetadataCollectionOption_Abandon)) + { + return S_OK; + } + + THROW_HR_IF(E_INVALIDARG, !outputFilePath); + + context->Complete(outputFilePath); + + return S_OK; + } + CATCH_RETURN() + + WINGET_UTIL_API WinGetMergeInstallerMetadata( + WINGET_STRING inputJSON, + WINGET_STRING_OUT* outputJSON, + UINT32 maximumOutputSizeInBytes, + WINGET_STRING logFilePath, + WinGetMergeInstallerMetadataOptions) try + { + THROW_HR_IF(E_INVALIDARG, !inputJSON); + THROW_HR_IF(E_INVALIDARG, !outputJSON); + + std::wstring merged = InstallerMetadataCollectionContext::Merge(inputJSON, maximumOutputSizeInBytes, GetPathOrEmpty(logFilePath)); + *outputJSON = ::SysAllocString(merged.c_str()); + + return S_OK; + } + CATCH_RETURN() } diff --git a/src/WinGetUtil/Source.def b/src/WinGetUtil/Source.def @@ -18,3 +18,6 @@ EXPORTS WinGetCreateManifest WinGetCloseManifest WinGetValidateManifestV3 + WinGetBeginInstallerMetadataCollection + WinGetCompleteInstallerMetadataCollection + WinGetMergeInstallerMetadata diff --git a/src/WinGetUtil/WinGetUtil.h b/src/WinGetUtil/WinGetUtil.h @@ -216,4 +216,60 @@ extern "C" WINGET_STRING versionA, WINGET_STRING versionB, INT* comparisonResult); + + // A handle to the metadata collection object. + typedef void* WINGET_INSTALLER_METADATA_COLLECTION_HANDLE; + + // Option flags for WinGetBeginInstallerMetadataCollection. + enum WinGetBeginInstallerMetadataCollectionOptions + { + WinGetBeginInstallerMetadataCollectionOption_None = 0, + // The inputJSON is a local file path, not a JSON string. + WinGetBeginInstallerMetadataCollectionOption_InputIsFilePath = 0x1, + // The inputJSON is a remote URI, not a JSON string. + WinGetBeginInstallerMetadataCollectionOption_InputIsURI = 0x2, + }; + + DEFINE_ENUM_FLAG_OPERATORS(WinGetBeginInstallerMetadataCollectionOptions); + + // Begins the installer metadata collection process. + // By default, inputJSON is expected to be a JSON string. See the WinGetBeginInstallerMetadataCollectionOptions for more options. + // logFilePath optionally specifies where to write the log file for the collection operation. + // The collectionHandle is owned by the caller and must be passed to WinGetCompleteInstallerMetadataCollection to free it. + WINGET_UTIL_API WinGetBeginInstallerMetadataCollection( + WINGET_STRING inputJSON, + WINGET_STRING logFilePath, + WinGetBeginInstallerMetadataCollectionOptions options, + WINGET_INSTALLER_METADATA_COLLECTION_HANDLE* collectionHandle); + + // Option flags for WinGetCompleteInstallerMetadataCollection. + enum WinGetCompleteInstallerMetadataCollectionOptions + { + WinGetCompleteInstallerMetadataCollectionOption_None = 0, + // Complete will simply free the collection handle without doing any additional work. + WinGetCompleteInstallerMetadataCollectionOption_Abandon = 0x1, + }; + + DEFINE_ENUM_FLAG_OPERATORS(WinGetCompleteInstallerMetadataCollectionOptions); + + // Completes the installer metadata collection process. + // Always frees the collectionHandle; WinGetCompleteInstallerMetadataCollection must be called exactly once for each call to WinGetBeginInstallerMetadataCollection. + WINGET_UTIL_API WinGetCompleteInstallerMetadataCollection( + WINGET_INSTALLER_METADATA_COLLECTION_HANDLE collectionHandle, + WINGET_STRING outputFilePath, + WinGetCompleteInstallerMetadataCollectionOptions options); + + // Option flags for WinGetMergeInstallerMetadata. + enum WinGetMergeInstallerMetadataOptions + { + WinGetMergeInstallerMetadataOptions_None = 0, + }; + + // Merges the given JSON metadata documents into a single one. + WINGET_UTIL_API WinGetMergeInstallerMetadata( + WINGET_STRING inputJSON, + WINGET_STRING_OUT* outputJSON, + UINT32 maximumOutputSizeInBytes, + WINGET_STRING logFilePath, + WinGetMergeInstallerMetadataOptions options); } diff --git a/src/WinGetUtil/WinGetUtil.vcxproj b/src/WinGetUtil/WinGetUtil.vcxproj @@ -162,9 +162,9 @@ <ClCompile> <Optimization>Disabled</Optimization> <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD</PreprocessorDefinitions> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">true</TreatWarningAsError> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">true</TreatWarningAsError> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</TreatWarningAsError> @@ -206,7 +206,7 @@ <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'"> <ClCompile> <PreprocessorDefinitions>WIN32;%(PreprocessorDefinitions);CLICOREDLLBUILD</PreprocessorDefinitions> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;$(ProjectDir)..\JsonCppLib\json;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</TreatWarningAsError> <ControlFlowGuard Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</ControlFlowGuard> <SDLCheck Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</SDLCheck> @@ -228,10 +228,10 @@ <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions);CLICOREDLLBUILD</PreprocessorDefinitions> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerCommonCore\Public;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(ProjectDir);$(ProjectDir)..\AppInstallerCommonCore;$(ProjectDir)..\AppInstallerRepositoryCore;$(ProjectDir)..\AppInstallerRepositoryCore\Public;$(ProjectDir)..\AppInstallerCommonCore\Public;$(ProjectDir)..\cpprestsdk\cpprestsdk\Release\include;$(ProjectDir)..\JsonCppLib;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">true</TreatWarningAsError> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">true</TreatWarningAsError> <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</TreatWarningAsError>