commit 685a06afd533db22374cf93a00e7cdb63e098240 parent d54c5a3d438b25289e574d305b9b2e8881b27b59 Author: JohnMcPMS <johnmcp@microsoft.com> Date: Mon, 10 Oct 2022 14:12:29 -0700 Enable mechanism for some control over correlation (#2577) ## Change The primary change is to enable some control over the correlation functionality so that different situations can behave differently. Specifically, the utility functionality of metadata collection, which is intended to be run in a "clean room" environment, can assume that a single change to ARP represents the target entry. In addition, this change enables diagnostic information to be output by the metadata collection. This includes the reason that an entry was chosen, and the confidence value for various possible entries if the heuristic matching is attempted. Finally, the correlation testbed is enhanced to run the metadata collection, allowing it to be tested alongside the inline correlation. The scripts have various improvements, such as a `-Wait` to leave the sandbox active and forcing the CSV to be output with a UTF8-BOM (Excel requires the BOM in order to open the CSV as UTF8). Diffstat:
10 files changed, 389 insertions(+), 71 deletions(-)
diff --git a/src/AppInstallerCLITests/Correlation.cpp b/src/AppInstallerCLITests/Correlation.cpp @@ -113,7 +113,8 @@ ResultSummary EvaluateDataSetWithHeuristic(const DataSet& dataSet, IARPMatchConf for (const auto& testCase : dataSet.TestCases) { arpEntries.push_back(GetARPEntryFromTestCase(testCase, /* isNew */ true)); - auto match = FindARPEntryForNewlyInstalledPackageWithHeuristics(GetManifestFromTestCase(testCase), arpEntries, correlationAlgorithm); + ARPHeuristicsCorrelationResult correlationResult = FindARPEntryForNewlyInstalledPackageWithHeuristics(GetManifestFromTestCase(testCase), arpEntries, correlationAlgorithm); + auto match = correlationResult.Package; arpEntries.pop_back(); if (match) diff --git a/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp @@ -229,7 +229,7 @@ namespace struct TestARPCorrelationData : public ARPCorrelationData { - ARPCorrelationResult CorrelateForNewlyInstalled(const Manifest::Manifest&) override + ARPCorrelationResult CorrelateForNewlyInstalled(const Manifest::Manifest&, const ARPCorrelationSettings&) override { return CorrelateForNewlyInstalledResult; } diff --git a/src/AppInstallerRepositoryCore/ARPCorrelation.cpp b/src/AppInstallerRepositoryCore/ARPCorrelation.cpp @@ -17,6 +17,7 @@ namespace AppInstaller::Repository::Correlation namespace { constexpr double MatchingThreshold = 0.5; + constexpr double MinimumDifferentiationThreshold = 0.05; IARPMatchConfidenceAlgorithm& InstanceInternal(std::optional<IARPMatchConfidenceAlgorithm*> algorithmOverride = {}) { @@ -57,7 +58,7 @@ namespace AppInstaller::Repository::Correlation #endif // Find the best match using heuristics - std::shared_ptr<IPackageVersion> FindARPEntryForNewlyInstalledPackageWithHeuristics( + ARPHeuristicsCorrelationResult FindARPEntryForNewlyInstalledPackageWithHeuristics( const Manifest::Manifest& manifest, const std::vector<ARPEntry>& arpEntries) { @@ -65,46 +66,52 @@ namespace AppInstaller::Repository::Correlation return FindARPEntryForNewlyInstalledPackageWithHeuristics(manifest, arpEntries, IARPMatchConfidenceAlgorithm::Instance()); } - std::shared_ptr<IPackageVersion> FindARPEntryForNewlyInstalledPackageWithHeuristics( + ARPHeuristicsCorrelationResult FindARPEntryForNewlyInstalledPackageWithHeuristics( const AppInstaller::Manifest::Manifest& manifest, const std::vector<ARPEntry>& arpEntries, IARPMatchConfidenceAlgorithm& algorithm) { + if (arpEntries.empty()) + { + AICLI_LOG(Repo, Warning, << "Empty ARP entries given"); + return {}; + } + AICLI_LOG(Repo, Verbose, << "Looking for best match in ARP for manifest " << manifest.Id); algorithm.Init(manifest); - std::optional<ARPEntry> bestMatch; - double bestScore = 0; + ARPHeuristicsCorrelationResult result; + result.Measures.reserve(arpEntries.size()); for (const auto& arpEntry : arpEntries) { auto score = algorithm.ComputeConfidence(arpEntry); AICLI_LOG(Repo, Verbose, << "Match confidence for " << arpEntry.Entry->GetProperty(PackageProperty::Id) << ": " << score); - if (score < MatchingThreshold) - { - AICLI_LOG(Repo, Verbose, << "Score is lower than threshold"); - continue; - } - - if (!bestMatch || bestScore < score) - { - bestMatch = arpEntry; - bestScore = score; - } + result.Measures.emplace_back(CorrelationMeasure{ score, arpEntry.Entry->GetInstalledVersion() }); } - if (bestMatch) + std::sort(result.Measures.begin(), result.Measures.end(), [](const CorrelationMeasure& a, const CorrelationMeasure& b) { return a.Measure > b.Measure; }); + + if (result.Measures[0].Measure < MatchingThreshold) { - AICLI_LOG(Repo, Verbose, << "Best match is " << bestMatch->Entry->GetProperty(PackageProperty::Id)); + AICLI_LOG(Repo, Verbose, << "Maximum score [" << result.Measures[0].Measure << "] is lower than threshold [" << MatchingThreshold << "]"); + result.Reason = "maximum score below threshold"; + } + else if (result.Measures.size() >= 2 && (result.Measures[0].Measure - result.Measures[1].Measure) < MinimumDifferentiationThreshold) + { + AICLI_LOG(Repo, Verbose, << "Top two scores, [" << result.Measures[0].Measure << "] and [" << result.Measures[1].Measure << "] are not significantly different [" << MinimumDifferentiationThreshold << "]"); + result.Reason = "top two scores are not significantly different"; } else { - AICLI_LOG(Repo, Verbose, << "No ARP entry had a correlation score surpassing the required threshold"); + AICLI_LOG(Repo, Verbose, << "Best match is " << result.Measures[0].Package->GetProperty(PackageVersionProperty::Id)); + result.Package = result.Measures[0].Package; + result.Reason = "heuristics match"; } - return bestMatch ? bestMatch->Entry->GetInstalledVersion() : nullptr; + return result; } void ARPCorrelationData::CapturePreInstallSnapshot() @@ -146,19 +153,12 @@ namespace AppInstaller::Repository::Correlation installed->GetProperty(PackageVersionProperty::Channel)); auto itr = std::lower_bound(m_preInstallSnapshot.begin(), m_preInstallSnapshot.end(), entryKey); - if (itr == m_preInstallSnapshot.end() || *itr != entryKey) - { - m_postInstallSnapshot.emplace_back(entry.Package, true); - } - else - { - m_postInstallSnapshot.emplace_back(entry.Package, false); - } + m_postInstallSnapshot.emplace_back(entry.Package, itr == m_preInstallSnapshot.end() || *itr != entryKey); } } } - ARPCorrelationResult ARPCorrelationData::CorrelateForNewlyInstalled(const Manifest::Manifest& manifest) + ARPCorrelationResult ARPCorrelationData::CorrelateForNewlyInstalled(const Manifest::Manifest& manifest, const ARPCorrelationSettings& settings) { AICLI_LOG(Repo, Verbose, << "Finding ARP entry matching newly installed package"); @@ -211,7 +211,7 @@ namespace AppInstaller::Repository::Correlation } // Add each ProductCode and UpgradeCode only once; - if (!appsAndFeaturesEntry.ProductCode.empty() && upgradeCodes.insert(appsAndFeaturesEntry.ProductCode).second) + if (!appsAndFeaturesEntry.ProductCode.empty() && productCodes.insert(appsAndFeaturesEntry.ProductCode).second) { manifestSearchRequest.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, appsAndFeaturesEntry.ProductCode)); } @@ -264,14 +264,21 @@ namespace AppInstaller::Repository::Correlation result.CountOfIntersectionOfChangesAndMatches = packagesInBoth.size(); // If there is only a single common package (changed and matches), it is almost certainly the correct one. - if (packagesInBoth.size() == 1) + if (settings.AllowNormalization && packagesInBoth.size() == 1) { result.Package = packagesInBoth[0]->GetInstalledVersion(); + result.Reason = "normalization match and new/changed"; } // If it wasn't changed but we still find a match, that is the best thing to report. - else if (findByManifest.Matches.size() == 1) + else if (settings.AllowNormalization && findByManifest.Matches.size() == 1) { result.Package = findByManifest.Matches[0].Package->GetInstalledVersion(); + result.Reason = "normalization match (not new/changed)"; + } + else if (settings.AllowSingleChange && result.ChangesToARP == 1) + { + result.Package = std::find_if(m_postInstallSnapshot.begin(), m_postInstallSnapshot.end(), [](const ARPEntry& e) { return e.IsNewOrUpdated; })->Entry->GetInstalledVersion(); + result.Reason = "only new/changed value"; } else { @@ -279,7 +286,7 @@ namespace AppInstaller::Repository::Correlation // to try and match the package with some ARP entry by assigning them scores. AICLI_LOG(Repo, Verbose, << "No exact ARP match found. Trying to find one with heuristics"); - result.Package = FindARPEntryForNewlyInstalledPackageWithHeuristics(manifest, m_postInstallSnapshot); + result = FindARPEntryForNewlyInstalledPackageWithHeuristics(manifest, m_postInstallSnapshot); } return result; diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -65,9 +65,23 @@ namespace AppInstaller::Repository::Metadata utility::string_t Status = L"status"; utility::string_t Metadata = L"metadata"; utility::string_t Diagnostics = L"diagnostics"; - + }; + + struct DiagnosticFields + { + // Eerror case utility::string_t ErrorHR = L"errorHR"; utility::string_t ErrorText = L"errorText"; + + // Non-error case + utility::string_t Reason = L"reason"; + utility::string_t ChangedEntryCount = L"changedEntryCount"; + utility::string_t MatchedEntryCount = L"matchedEntryCount"; + utility::string_t IntersectionCount = L"intersectionCount"; + utility::string_t CorrelationMeasures = L"correlationMeasures"; + utility::string_t Value = L"value"; + utility::string_t Name = L"name"; + utility::string_t Publisher = L"publisher"; }; std::string GetRequiredString(const web::json::value& value, const utility::string_t& field) @@ -736,7 +750,11 @@ namespace AppInstaller::Repository::Metadata // Copy the metadata from the current; this function takes care of moving data to historical if the submission is new. m_outputMetadata.CopyFrom(m_currentMetadata, m_submissionIdentifier); - Correlation::ARPCorrelationResult correlationResult = m_correlationData->CorrelateForNewlyInstalled(m_incomingManifest); + Correlation::ARPCorrelationSettings settings; + // As this code is typically run in a controlled environment, we can assume that a single value change is very likely the correct value. + settings.AllowSingleChange = true; + + Correlation::ARPCorrelationResult correlationResult = m_correlationData->CorrelateForNewlyInstalled(m_incomingManifest, settings); if (correlationResult.Package) { @@ -814,9 +832,31 @@ namespace AppInstaller::Repository::Metadata else { m_outputStatus = OutputStatus::LowConfidence; + } - // TODO: Output diagnostics such as the top 10 entries by confidence. + // Create the diagnostics data, based on the other values from the correlation result. + DiagnosticFields fields; + + m_outputDiagnostics[fields.Reason] = AppInstaller::JSON::GetStringValue(correlationResult.Reason); + m_outputDiagnostics[fields.ChangedEntryCount] = web::json::value::number(static_cast<int64_t>(correlationResult.ChangesToARP)); + m_outputDiagnostics[fields.MatchedEntryCount] = web::json::value::number(static_cast<int64_t>(correlationResult.MatchesInARP)); + m_outputDiagnostics[fields.IntersectionCount] = web::json::value::number(static_cast<int64_t>(correlationResult.CountOfIntersectionOfChangesAndMatches)); + + constexpr size_t MaximumDiagnosticMeasures = 10; + web::json::value measuresArray = web::json::value::array(); + for (size_t i = 0; i < correlationResult.Measures.size() && i < MaximumDiagnosticMeasures; ++i) + { + web::json::value measureValue; + const auto& measure = correlationResult.Measures[i]; + + measureValue[fields.Value] = web::json::value::number(measure.Measure); + measureValue[fields.Name] = AppInstaller::JSON::GetStringValue(measure.Package->GetProperty(PackageVersionProperty::Name)); + measureValue[fields.Publisher] = AppInstaller::JSON::GetStringValue(measure.Package->GetProperty(PackageVersionProperty::Publisher)); + + measuresArray[i] = std::move(measureValue); } + + m_outputDiagnostics[fields.CorrelationMeasures] = std::move(measuresArray); } void InstallerMetadataCollectionContext::ParseInputJson_1_0(web::json::value& input) @@ -968,6 +1008,7 @@ namespace AppInstaller::Repository::Metadata AICLI_LOG(Repo, Info, << "Setting error JSON 1.0 fields"); OutputFields_1_0 fields; + DiagnosticFields diagnosticFields; web::json::value result; @@ -979,8 +1020,8 @@ namespace AppInstaller::Repository::Metadata web::json::value error; - error[fields.ErrorHR] = web::json::value::number(static_cast<int64_t>(m_errorHR)); - error[fields.ErrorText] = AppInstaller::JSON::GetStringValue(m_errorText); + error[diagnosticFields.ErrorHR] = web::json::value::number(static_cast<int64_t>(m_errorHR)); + error[diagnosticFields.ErrorText] = AppInstaller::JSON::GetStringValue(m_errorText); result[fields.Diagnostics] = std::move(error); diff --git a/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h b/src/AppInstallerRepositoryCore/Public/winget/ARPCorrelation.h @@ -42,16 +42,65 @@ namespace AppInstaller::Repository::Correlation bool IsNewOrUpdated; }; - struct ARPCorrelationResult + // One of the possible options that could be chosen for correlation. + struct CorrelationMeasure + { + // The value that the correlation algorithm assigned to the match with the package. + double Measure{}; + + // The package that was measured. + std::shared_ptr<AppInstaller::Repository::IPackageVersion> Package{}; + }; + + // The result of a heuristics correlation attempt. + struct ARPHeuristicsCorrelationResult { // Correlated package from ARP std::shared_ptr<AppInstaller::Repository::IPackageVersion> Package{}; + + // The reason for the correlation (for diagnostics). + std::string Reason; + + // The correlation metrics and their associated ARP package information (for diagnostics). + std::vector<CorrelationMeasure> Measures; + }; + + // The result of a correlation attempt. + struct ARPCorrelationResult : public ARPHeuristicsCorrelationResult + { // Number of ARP entries that are new or updated size_t ChangesToARP{}; + // Number of ARP entries that match with the installed package size_t MatchesInARP{}; + // Number of changed ARP entries that match the installed package size_t CountOfIntersectionOfChangesAndMatches{}; + + ARPCorrelationResult& operator=(ARPHeuristicsCorrelationResult&& other) + { + *static_cast<ARPHeuristicsCorrelationResult*>(this) = std::move(other); + return *this; + } + }; + + // Allows callers finer control over how the correlation result will be chosen. + // The values appear in order of their application in the correlation algorithm, meaning that a later + // setting that is set to true can be pre-empted by an earlier setting, if a correlation occurs with the + // earlier setting. + // The default values are chosen to reflect what is used after an install on a consumer system. + struct ARPCorrelationSettings + { + // This setting controls whether the name and publisher normalization algorithm will be used for correlation. + // When true, normalization will be the first choice for correlation. This means that a normalized name+publisher + // match will result in correlation (unless there are multiple matches). + // When false, normalization will only be used for the statistics (MatchesInARP), but the correlation result package + // will not be based on normalization. + bool AllowNormalization = true; + + // This settings controls whether a single changed ARP entry is sufficient to result in correlation. + // When true, if only a single ARP entry is detected as new or changed, it will be chosen as the correlated result. + bool AllowSingleChange = false; }; struct IARPMatchConfidenceAlgorithm @@ -70,11 +119,11 @@ namespace AppInstaller::Repository::Correlation #endif }; - std::shared_ptr<AppInstaller::Repository::IPackageVersion> FindARPEntryForNewlyInstalledPackageWithHeuristics( + ARPHeuristicsCorrelationResult FindARPEntryForNewlyInstalledPackageWithHeuristics( const AppInstaller::Manifest::Manifest& manifest, const std::vector<ARPEntry>& arpEntries); - std::shared_ptr<AppInstaller::Repository::IPackageVersion> FindARPEntryForNewlyInstalledPackageWithHeuristics( + ARPHeuristicsCorrelationResult FindARPEntryForNewlyInstalledPackageWithHeuristics( const AppInstaller::Manifest::Manifest& manifest, const std::vector<ARPEntry>& arpEntries, IARPMatchConfidenceAlgorithm& algorithm); @@ -92,7 +141,7 @@ namespace AppInstaller::Repository::Correlation void CapturePostInstallSnapshot(); // Correlates the given manifest against the data previously collected with capture calls. - virtual ARPCorrelationResult CorrelateForNewlyInstalled(const Manifest::Manifest& manifest); + virtual ARPCorrelationResult CorrelateForNewlyInstalled(const Manifest::Manifest& manifest, const ARPCorrelationSettings& settings = {}); const std::vector<ARPEntrySnapshot>& GetPreInstallSnapshot() const { return m_preInstallSnapshot; } diff --git a/tools/CorrelationTestbed/InSandboxScript.ps1 b/tools/CorrelationTestbed/InSandboxScript.ps1 @@ -4,7 +4,9 @@ Param( [String] $PackageIdentifier, [String] $SourceName, [String] $OutputPath, - [Switch] $UseDev + [Switch] $UseDev, + [Switch] $MetadataCollection, + [String] $System32Path ) function Get-ARPTable { @@ -81,6 +83,12 @@ if ($UseDev) $installAndCorrelationExpression = -join($installAndCorrelationExpression, ' -dev') } +if ($MetadataCollection) +{ + $wingetUtilPath = Join-Path $PSScriptRoot "WinGetUtil.dll" + $installAndCorrelationExpression = -join($installAndCorrelationExpression, ' -meta "', $wingetUtilPath, '" -sys32 "', $System32Path,'"') +} + Invoke-Expression $installAndCorrelationExpression Write-Host @" diff --git a/tools/CorrelationTestbed/InstallAndCheckCorrelation/InstallAndCheckCorrelation/InstallAndCheckCorrelation.cpp b/tools/CorrelationTestbed/InstallAndCheckCorrelation/InstallAndCheckCorrelation/InstallAndCheckCorrelation.cpp @@ -13,6 +13,9 @@ #include <iostream> #include <filesystem> #include <fstream> +#include <sstream> + +#include <WinGetUtil.h> using namespace std::string_view_literals; using namespace winrt::Microsoft::Management::Deployment; @@ -31,33 +34,46 @@ struct JSONPair }; template <typename T> -struct JSONQuote +struct JSONControl +{ + constexpr static bool quote = true; + constexpr static bool output = true; +}; + +template <> +struct JSONControl<HRESULT> { - constexpr static bool value = true; + constexpr static bool quote = false; + constexpr static bool output = true; }; template <> -struct JSONQuote<HRESULT> +struct JSONControl<bool> { - constexpr static bool value = false; + constexpr static bool quote = false; + constexpr static bool output = true; }; template <> -struct JSONQuote<bool> +struct JSONControl<nullptr_t> { - constexpr static bool value = false; + constexpr static bool quote = false; + constexpr static bool output = false; }; template <typename T> std::ostream& operator<<(std::ostream& out, const JSONPair<T>& pair) { out << '"' << pair.Name << "\": "; - if (JSONQuote<T>::value) + if (JSONControl<T>::quote) { out << '"'; } - out << pair.Value; - if (JSONQuote<T>::value) + if (JSONControl<T>::output) + { + out << pair.Value; + } + if (JSONControl<T>::quote) { out << '"'; } @@ -128,6 +144,9 @@ struct Main std::string packageIdentifier; std::string sourceName; std::filesystem::path outputPath; + bool metadataCollection = false; + std::filesystem::path wingetUtilPath; + std::filesystem::path sys32Path; bool useDevCLSIDs = false; bool onlyCorrelate = false; @@ -154,6 +173,15 @@ struct Main { outputPath = argv[++i]; } + else if ("-meta"sv == argv[i] && i + 1 < argc) + { + metadataCollection = true; + wingetUtilPath = argv[++i]; + } + else if ("-sys32"sv == argv[i] && i + 1 < argc) + { + sys32Path = argv[++i]; + } else if ("-dev"sv == argv[i]) { useDevCLSIDs = true; @@ -266,6 +294,99 @@ struct Main error = "A source name must be supplied, use -src"; return; } + + if (metadataCollection && sys32Path.empty()) + { + hr = E_INVALIDARG; + error = "Metadata collection requires mapping in the host's System32"; + return; + } + } + + using WinGetBeginInstallerMetadataCollectionPtr = HRESULT (__stdcall *)( + WINGET_STRING inputJSON, + WINGET_STRING logFilePath, + WinGetBeginInstallerMetadataCollectionOptions options, + WINGET_INSTALLER_METADATA_COLLECTION_HANDLE* collectionHandle); + + using WinGetCompleteInstallerMetadataCollectionPtr = HRESULT(__stdcall*)( + WINGET_INSTALLER_METADATA_COLLECTION_HANDLE collectionHandle, + WINGET_STRING outputFilePath, + WinGetCompleteInstallerMetadataCollectionOptions options); + + WinGetBeginInstallerMetadataCollectionPtr WinGetBeginInstallerMetadataCollection = nullptr; + WinGetCompleteInstallerMetadataCollectionPtr WinGetCompleteInstallerMetadataCollection = nullptr; + WINGET_INSTALLER_METADATA_COLLECTION_HANDLE MetadataCollectionHandle = nullptr; + + void LoadWingetUtil() + { + AddDllDirectory(sys32Path.wstring().c_str()); + HMODULE wingetutilModule = LoadLibraryExW(wingetUtilPath.wstring().c_str(), nullptr, LOAD_LIBRARY_SEARCH_USER_DIRS); + if (!wingetutilModule) + { + hr = HRESULT_FROM_WIN32(GetLastError()); + return; + } + + this->WinGetBeginInstallerMetadataCollection = reinterpret_cast<WinGetBeginInstallerMetadataCollectionPtr>(GetProcAddress(wingetutilModule, "WinGetBeginInstallerMetadataCollection")); + if (!this->WinGetBeginInstallerMetadataCollection) + { + hr = HRESULT_FROM_WIN32(GetLastError()); + return; + } + + this->WinGetCompleteInstallerMetadataCollection = reinterpret_cast<WinGetCompleteInstallerMetadataCollectionPtr>(GetProcAddress(wingetutilModule, "WinGetCompleteInstallerMetadataCollection")); + if (!this->WinGetCompleteInstallerMetadataCollection) + { + hr = HRESULT_FROM_WIN32(GetLastError()); + return; + } + } + + void BeginMetadataCollection() + { + std::filesystem::path metadataInputPath = outputPath.parent_path(); + metadataInputPath /= "metadata_input.json"; + std::ofstream stream{ metadataInputPath }; + + stream << "{" << std::endl; + stream << JSONPair{ "supportedMetadataVersion", "1.1"}; + // TODO: Could theoretically produce this if we could enumerate the data via COM + // stream << JSONPair{ "currentMetadata", "" }; + stream << JSONPair{ "submissionData", nullptr, false } << "\n{\n"; + stream << JSONPair{ "submissionIdentifier", packageIdentifier, false }; + stream << "\n},\n"; + stream << JSONPair{ "packageData", nullptr, false } << "\n{\n"; + stream << JSONPair{ "installerHash", "none" }; + stream << JSONPair{ "DefaultLocale", nullptr, false } << "\n{\n"; + stream << JSONPair{ "PackageLocale", "x-neutral" }; + stream << JSONPair{ "PackageName", packageName }; + stream << JSONPair{ "Publisher", packagePublisher, false }; + stream << "\n}\n"; + stream << "\n},\n"; + // Keep at the end to prevent a dangling comma + stream << JSONPair{ "version", "1.0", false } << "}" << std::endl; + + std::filesystem::path metadataLogPath = outputPath.parent_path(); + metadataLogPath /= "metadata_log.txt"; + + hr = this->WinGetBeginInstallerMetadataCollection(metadataInputPath.wstring().c_str(), metadataLogPath.wstring().c_str(), WinGetBeginInstallerMetadataCollectionOption_InputIsFilePath, &MetadataCollectionHandle); + if (FAILED(hr)) + { + return; + } + } + + void CompleteMetadataCollection() + { + std::filesystem::path metadataOutputPath = outputPath.parent_path(); + metadataOutputPath /= "metadata_output.json"; + + hr = this->WinGetCompleteInstallerMetadataCollection(MetadataCollectionHandle, metadataOutputPath.wstring().c_str(), WinGetCompleteInstallerMetadataCollectionOption_None); + if (FAILED(hr)) + { + return; + } } void Install() @@ -306,7 +427,9 @@ struct Main if (findResult.Status() != FindPackagesResultStatus::Ok) { hr = E_FAIL; - error = "Error finding packages"; + std::ostringstream stream; + stream << "Error " << static_cast<int>(findResult.Status()) << " finding package"; + error = std::move(stream).str(); return; } @@ -325,10 +448,15 @@ struct Main action = "Inspect package"; auto installVersion = package.DefaultInstallVersion(); packageName = ConvertToUTF8(installVersion.DisplayName()); - if (useDevCLSIDs) + packagePublisher = ConvertToUTF8(installVersion.Publisher()); + + if (metadataCollection) { - // Publisher is not yet available on the release version; make this unconditional when it is - packagePublisher = ConvertToUTF8(installVersion.Publisher()); + BeginMetadataCollection(); + if (FAILED(hr)) + { + return; + } } if (!onlyCorrelate) @@ -340,6 +468,7 @@ struct Main installOptions.PackageInstallMode(PackageInstallMode::Silent); std::cout << "Beginning to install " << packageIdentifier << " (" << packageName << ") from " << sourceName << "..." << std::endl; + action = "Install package"; auto installOperation = packageManager.InstallPackageAsync(package, installOptions); if (installOperation.wait_for(std::chrono::minutes(10)) != AsyncStatus::Completed) @@ -357,6 +486,15 @@ struct Main return; } } + + if (metadataCollection) + { + CompleteMetadataCollection(); + if (FAILED(hr)) + { + return; + } + } } catch (const winrt::hresult_error& hre) { @@ -435,11 +573,7 @@ struct Main { correlatePackageKnown = true; packageKnownName = ConvertToUTF8(installed.DisplayName()); - if (useDevCLSIDs) - { - // Publisher is not yet available on the release version; make this unconditional when it is - packageKnownPublisher = ConvertToUTF8(installed.Publisher()); - } + packageKnownPublisher = ConvertToUTF8(installed.Publisher()); } } catch (const winrt::hresult_error& hre) @@ -520,11 +654,7 @@ struct Main { correlateArchive = true; archiveName = ConvertToUTF8(installed.DisplayName()); - if (useDevCLSIDs) - { - // Publisher is not yet available on the release version; make this unconditional when it is - archivePublisher = ConvertToUTF8(installed.Publisher()); - } + archivePublisher = ConvertToUTF8(installed.Publisher()); break; } } @@ -575,6 +705,15 @@ struct Main return; } + if (metadataCollection) + { + LoadWingetUtil(); + if (FAILED(hr)) + { + return; + } + } + auto co_uninitialize = wil::CoInitializeEx(); // Execute the install step diff --git a/tools/CorrelationTestbed/InstallAndCheckCorrelation/InstallAndCheckCorrelation/InstallAndCheckCorrelation.vcxproj b/tools/CorrelationTestbed/InstallAndCheckCorrelation/InstallAndCheckCorrelation/InstallAndCheckCorrelation.vcxproj @@ -90,6 +90,7 @@ <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <ConformanceMode>true</ConformanceMode> <LanguageStandard>stdcpp17</LanguageStandard> + <AdditionalIncludeDirectories>$(SolutionDir)..\..\..\src\WinGetUtil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -105,6 +106,7 @@ <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <ConformanceMode>true</ConformanceMode> <LanguageStandard>stdcpp17</LanguageStandard> + <AdditionalIncludeDirectories>$(SolutionDir)..\..\..\src\WinGetUtil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -120,6 +122,7 @@ <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <ConformanceMode>true</ConformanceMode> <LanguageStandard>stdcpp17</LanguageStandard> + <AdditionalIncludeDirectories>$(SolutionDir)..\..\..\src\WinGetUtil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -136,6 +139,7 @@ <ConformanceMode>true</ConformanceMode> <LanguageStandard>stdcpp17</LanguageStandard> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> + <AdditionalIncludeDirectories>$(SolutionDir)..\..\..\src\WinGetUtil;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> <SubSystem>Console</SubSystem> diff --git a/tools/CorrelationTestbed/Process-CorrelationResults.ps1 b/tools/CorrelationTestbed/Process-CorrelationResults.ps1 @@ -24,6 +24,8 @@ $stats = @{ Failed = 0 CorrelatePackageKnown = 0 CorrelateArchive = 0 + CorrelateMetadata = 0 + CorrelationDisagreement = 0 } # Aggregate results in a single CSV file @@ -44,17 +46,35 @@ foreach ($result in (Get-ChildItem $ResultsPath -Directory)) continue } + $metadataJSON = Join-Path $result.FullName "metadata_output.json" + if (Test-Path $metadataJSON) + { + $metadataObj = (Get-Content -Path $metadataJSON -Encoding utf8 | ConvertFrom-Json) + } + if ($resultObj.HRESULT -eq 0) { $stats.Completed++ $stats.CorrelateArchive += $resultObj.CorrelateArchive $stats.CorrelatePackageKnown += $resultObj.CorrelatePackageKnown - Export-Csv -InputObject ($resultObj | Select-Object -Property * -ExcludeProperty @("Error", "Phase", "Action", "HRESULT") ) -Path $resultFile -Append + if ($metadataObj -and $metadataObj.status -eq "Success") + { + $stats.CorrelateMetadata += 1 + Add-Member -InputObject $resultObj -MemberType NoteProperty -Name "CorrelateMetadata" -Value 1 -Force + Add-Member -InputObject $resultObj -MemberType NoteProperty -Name "MetadataName" -Value $metadataObj.metadata[0].metadata[0].AppsAndFeaturesEntries[0].DisplayName -Force + Add-Member -InputObject $resultObj -MemberType NoteProperty -Name "MetadataPublisher" -Value $metadataObj.metadata[0].metadata[0].AppsAndFeaturesEntries[0].Publisher -Force + + if ($resultObj.PackageKnownName -ne "" -and $resultObj.MetadataName -ne "" -and $resultObj.PackageKnownName -ne $resultObj.MetadataName) + { + $stats.CorrelationDisagreement += 1 + } + } + Export-Csv -InputObject ($resultObj | Select-Object -Property * -ExcludeProperty @("Error", "Phase", "Action", "HRESULT") ) -Path $resultFile -Append -Encoding UTF8BOM } else { $stats.Failed++ - Export-Csv -InputObject $resultObj -Path $failedFile -Append + Export-Csv -InputObject $resultObj -Path $failedFile -Append -Encoding UTF8BOM } } @@ -62,4 +82,6 @@ foreach ($result in (Get-ChildItem $ResultsPath -Directory)) $stats.CompletedRatio = $stats.Completed / $stats.Total $stats.CorrelateArchiveRatio = $stats.CorrelateArchive / $stats.Completed $stats.CorrelatePackageKnownRatio = $stats.CorrelatePackageKnown / $stats.Completed +$stats.CorrelateMetadataRatio = $stats.CorrelateMetadata / $stats.Completed +$stats.CorrelationDisagreementRatio = $stats.CorrelationDisagreement / $stats.Completed $stats | ConvertTo-Json | Out-File $statsFile -Force diff --git a/tools/CorrelationTestbed/Test-CorrelationInSandbox.ps1 b/tools/CorrelationTestbed/Test-CorrelationInSandbox.ps1 @@ -15,7 +15,13 @@ Param( [Parameter(HelpMessage = "The results output path.")] [String] $ResultsPath, [Parameter(HelpMessage = "The path to registry files that should be injected before the test.")] - [String] $RegFileDirectory + [String] $RegFileDirectory, + [Parameter(HelpMessage = "Indicates that the metadata collection process should be run.")] + [Switch] $MetadataCollection, + [Parameter(HelpMessage = "The path to WinGetUtil.dll; only the release build works.")] + [String] $WingetUtilPath, + [Parameter(HelpMessage = "Wait for user input before tearing down each sandbox.")] + [Switch] $Wait ) $ErrorActionPreference = "Stop" @@ -65,6 +71,26 @@ Either build the local dev package, or provide the location using -DevPackagePat } } +# Validate that WinGetUtil.dll exists if metadata collection is requested + +if ($MetadataCollection) +{ + if (-not $WingetUtilPath) + { + $WingetUtilPath = Join-Path $PSScriptRoot "..\..\src\x64\Debug\WinGetUtil\WinGetUtil.dll" + } + + $WingetUtilPath = [System.IO.Path]::GetFullPath($WingetUtilPath) + + if (-not (Test-Path $WingetUtilPath)) + { + Write-Error -Category InvalidArgument -Message @" +WinGetUtil.dll does not exist in the path $WingetUtilPath +Either build the binary, or provide the location using -WingetUtilPath +"@ + } +} + # Check if Windows Sandbox is enabled if (-Not (Get-Command 'WindowsSandbox' -ErrorAction SilentlyContinue)) @@ -249,6 +275,11 @@ if (-not $UseDev) Write-Host } +if ($MetadataCollection) +{ + Copy-Item -Path $WingetUtilPath -Destination $tempFolder -Force +} + # Copy main script $mainPs1FileName = 'InSandboxScript.ps1' @@ -262,6 +293,7 @@ foreach ($packageIdentifier in $PackageIdentifiers) New-Item -ItemType Directory $outPath | Out-Null $outPathInSandbox = Join-Path -Path $desktopInSandbox -ChildPath (Split-Path -Path $outPath -Leaf) + $system32PathInSandbox = Join-Path -Path $desktopInSandbox -ChildPath "hostSystem32" if ($UseDev) { @@ -272,7 +304,7 @@ foreach ($packageIdentifier in $PackageIdentifiers) $dependenciesPathsInSandbox = "@('$($vcLibsUwp.pathInSandbox)', '$($uiLibsUwp.pathInSandbox)')" } - $bootstrapPs1Content = ".\$mainPs1FileName -DesktopAppInstallerDependencyPath @($dependenciesPathsInSandbox) -PackageIdentifier '$packageIdentifier' -SourceName '$Source' -OutputPath '$outPathInSandbox'" + $bootstrapPs1Content = ".\$mainPs1FileName -DesktopAppInstallerDependencyPath @($dependenciesPathsInSandbox) -PackageIdentifier '$packageIdentifier' -SourceName '$Source' -OutputPath '$outPathInSandbox' -System32Path '$system32PathInSandbox'" if ($UseDev) { @@ -282,6 +314,11 @@ foreach ($packageIdentifier in $PackageIdentifiers) { $bootstrapPs1Content += " -DesktopAppInstallerPath '$($desktopAppInstaller.pathInSandbox)'" } + + if ($MetadataCollection) + { + $bootstrapPs1Content += " -MetadataCollection" + } $bootstrapPs1FileName = 'Bootstrap.ps1' $bootstrapPs1Content | Out-File (Join-Path $tempFolder $bootstrapPs1FileName) -Force @@ -331,6 +368,11 @@ foreach ($packageIdentifier in $PackageIdentifiers) <SandboxFolder>$exePathInSandbox</SandboxFolder> <ReadOnly>true</ReadOnly> </MappedFolder> + <MappedFolder> + <HostFolder>C:\Windows\System32</HostFolder> + <SandboxFolder>$system32PathInSandbox</SandboxFolder> + <ReadOnly>true</ReadOnly> + </MappedFolder> $devPackageXMLFragment $regFileDirXMLFragment <MappedFolder> @@ -373,6 +415,11 @@ foreach ($packageIdentifier in $PackageIdentifiers) Start-Sleep 1 } + if ($Wait) + { + Read-Host "Press Enter to close sandbox and continue..." + } + Close-WindowsSandbox }