commit 0d181f6415d6906d1ec79de66fd8644a4d37c443 parent 8eb122897afbf237589927d05b7146b97c5e87d8 Author: yao-msft <50888816+yao-msft@users.noreply.github.com> Date: Mon, 13 Mar 2023 17:45:41 -0700 Add installed files and startup link metadata collection in Installer Metadata collection util (#2985) Diffstat:
21 files changed, 1096 insertions(+), 41 deletions(-)
diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt @@ -277,6 +277,7 @@ LEN Linux LLVM llvmorg +lnk LOCALAPPDATA localtime LOGONLYONERROR diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt @@ -234,14 +234,18 @@ NESTEDINSTALLER netlify Newtonsoft NOEXPAND +NOLINKINFO nonetwork NONFOLDERS nonterminated normer +NOSEARCH NOSEPARATOR notalostreference NOTAPROPERTY notmatch +NOTRACK +NOUPDATE npmjs nsis nuffing @@ -376,6 +380,7 @@ UChars ucnv uec uninitialize +unins uninstallation uninstaller uninstallprevious diff --git a/src/AppInstallerCLICore/Workflows/PromptFlow.cpp b/src/AppInstallerCLICore/Workflows/PromptFlow.cpp @@ -258,7 +258,7 @@ namespace AppInstaller::CLI::Workflow // When prompting for a single package, we use the provided location directly. // This is different from when we prompt for multiple packages or use the root in the settings. - context.Args.AddArg(Execution::Args::Type::InstallLocation, m_installLocation.string()); + context.Args.AddArg(Execution::Args::Type::InstallLocation, m_installLocation.u8string()); } void PromptForMultiplePackages(Execution::Context& context, std::vector<Execution::Context*>& packagesToPrompt) override @@ -308,7 +308,7 @@ namespace AppInstaller::CLI::Workflow auto installLocation = m_installLocation; installLocation += "\\" + packageId; AICLI_LOG(CLI, Info, << "Setting install location for package [" << packageId << "] to: " << installLocation); - context.Args.AddArg(Execution::Args::Type::InstallLocation, installLocation.string()); + context.Args.AddArg(Execution::Args::Type::InstallLocation, installLocation.u8string()); } std::filesystem::path m_installLocation; diff --git a/src/AppInstallerCLITests/FolderFileWatcher.cpp b/src/AppInstallerCLITests/FolderFileWatcher.cpp @@ -74,7 +74,7 @@ TEST_CASE("FolderFileWatcher_CreateNewFiles", "[FolderFileWatcher]") std::this_thread::sleep_for(100ms); folderFileWatcher.Stop(); - auto& watchedFiles = folderFileWatcher.files(); + auto& watchedFiles = folderFileWatcher.Files(); auto tempFile1RelativePath = RemoveRoot(dirToWatch, tempFile1.GetPath()); auto foundTempFile1 = watchedFiles.find(tempFile1RelativePath); @@ -105,7 +105,7 @@ TEST_CASE("FolderFileWatcher_CreateAfterStop", "[FolderFileWatcher]") TempFile tempFile2(dirToWatch.GetPath(), "file2_", ".txt"); WriteText(tempFile2.GetPath()); - auto& watchedFiles = folderFileWatcher.files(); + auto& watchedFiles = folderFileWatcher.Files(); auto tempFile1RelativePath = RemoveRoot(dirToWatch, tempFile1.GetPath()); auto foundTempFile1 = watchedFiles.find(tempFile1RelativePath); @@ -144,7 +144,7 @@ TEST_CASE("FolderFileWatcher_CreateNewFilesAndRename", "[FolderFileWatcher]") std::this_thread::sleep_for(100ms); folderFileWatcher.Stop(); - auto& watchedFiles = folderFileWatcher.files(); + auto& watchedFiles = folderFileWatcher.Files(); auto tempFile1RelativePath = RemoveRoot(dirToWatch, tempFile1Path); auto foundTempFile1 = watchedFiles.find(tempFile1RelativePath); @@ -198,7 +198,7 @@ TEST_CASE("FolderFileWatcher_CreateNewFilesAndDelete", "[FolderFileWatcher]") std::this_thread::sleep_for(100ms); folderFileWatcher.Stop(); - auto& watchedFiles = folderFileWatcher.files(); + auto& watchedFiles = folderFileWatcher.Files(); auto tempFile1RelativePath = RemoveRoot(dirToWatch, tempFile1.GetPath()); auto foundTempFile1 = watchedFiles.find(tempFile1RelativePath); @@ -243,7 +243,7 @@ TEST_CASE("FolderFileWatcher_Extension_CreateNewFiles", "[FolderFileWatcher]") std::this_thread::sleep_for(100ms); folderFileExtensionWatcher.Stop(); - auto& watchedFiles = folderFileExtensionWatcher.files(); + auto& watchedFiles = folderFileExtensionWatcher.Files(); auto tempFile1RelativePath = RemoveRoot(dirToWatch, tempFile1.GetPath()); auto foundTempFile1 = watchedFiles.find(tempFile1RelativePath); @@ -278,7 +278,7 @@ TEST_CASE("FolderFileWatcher_Extension_CreateAfterStop", "[FolderFileWatcher]") TempFile tempFile2(dirToWatch.GetPath(), "file2_", ".txt"); WriteText(tempFile2.GetPath()); - auto& watchedFiles = folderFileExtensionWatcher.files(); + auto& watchedFiles = folderFileExtensionWatcher.Files(); auto tempFile1RelativePath = RemoveRoot(dirToWatch, tempFile1.GetPath()); auto foundTempFile1 = watchedFiles.find(tempFile1RelativePath); @@ -317,7 +317,7 @@ TEST_CASE("FolderFileWatcher_Extension_CreateNewFilesAndRename", "[FolderFileWat std::this_thread::sleep_for(100ms); folderFileExtensionWatcher.Stop(); - auto& watchedFiles = folderFileExtensionWatcher.files(); + auto& watchedFiles = folderFileExtensionWatcher.Files(); auto tempFile1RelativePath = RemoveRoot(dirToWatch, tempFile1Path); auto foundTempFile1 = watchedFiles.find(tempFile1RelativePath); @@ -371,7 +371,7 @@ TEST_CASE("FolderFileWatcher_Extension_CreateNewFilesAndDelete", "[FolderFileWat std::this_thread::sleep_for(100ms); folderFileExtensionWatcher.Stop(); - auto& watchedFiles = folderFileExtensionWatcher.files(); + auto& watchedFiles = folderFileExtensionWatcher.Files(); auto tempFile1RelativePath = RemoveRoot(dirToWatch, tempFile1.GetPath()); auto foundTempFile1 = watchedFiles.find(tempFile1RelativePath); diff --git a/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp b/src/AppInstallerCLITests/InstallerMetadataCollectionContext.cpp @@ -237,9 +237,31 @@ namespace ARPCorrelationResult CorrelateForNewlyInstalledResult; }; + struct TestInstalledFilesCorrelation : public InstalledFilesCorrelation + { + Correlation::InstallationMetadata CorrelateForNewlyInstalled(const Manifest::Manifest&, const std::string&) override + { + return InstallationMetadata; + } + + void StartFileWatcher() override {} + + void StopFileWatcher() override {} + + Correlation::InstallationMetadata InstallationMetadata; + }; + + InstallerMetadataCollectionContext CreateTestContext( + std::unique_ptr<ARPCorrelationData>&& data, + std::unique_ptr<InstalledFilesCorrelation>&& installedFiles, + TestInput& input) + { + return { std::move(data), std::move(installedFiles), input.ToJSON() }; + } + InstallerMetadataCollectionContext CreateTestContext(std::unique_ptr<ARPCorrelationData>&& data, TestInput& input) { - return { std::move(data), input.ToJSON() }; + return { std::move(data), std::make_unique<TestInstalledFilesCorrelation>(), input.ToJSON()}; } InstallerMetadataCollectionContext CreateTestContext(TestInput& input) @@ -853,3 +875,298 @@ TEST_CASE("MetadataCollection_Merge_SameInstaller_Scopes", "[metadata_collection } } } + +TEST_CASE("MetadataCollection_NewPackage_1_2", "[metadata_collection]") +{ + TestInput input(MinimalDefaults); + input.SupportedMetadataVersion = "1.2"; + auto correlationData = std::make_unique<TestARPCorrelationData>(); + auto installedFilesData = std::make_unique<TestInstalledFilesCorrelation>(); + + Manifest::Manifest manifest; + manifest.DefaultLocalization.Add<Manifest::Localization::PackageName>("Test Package Name"); + manifest.DefaultLocalization.Add<Manifest::Localization::Publisher>("Test Publisher"); + manifest.Version = "1.2.3"; + manifest.Installers.push_back({}); + manifest.Installers[0].ProductCode = "{guid}"; + + IPackageVersion::Metadata metadata; + metadata[PackageVersionMetadata::InstalledType] = Manifest::InstallerTypeToString(Manifest::InstallerTypeEnum::Msi); + metadata[PackageVersionMetadata::InstalledScope] = Manifest::ScopeToString(Manifest::ScopeEnum::User); + + correlationData->CorrelateForNewlyInstalledResult.Package = std::make_shared<TestPackageVersion>(manifest, metadata); + + Correlation::InstallationMetadata installedFiles; + installedFiles.InstalledFiles.DefaultInstallLocation = "%TEMP%\\TestApp"; + Manifest::InstalledFile installedFile; + installedFile.RelativeFilePath = "test.exe"; + installedFile.FileSha256 = Utility::SHA256::ConvertToBytes("d2a45116709136462ee7a1c42f0e75f0efa258fe959b1504dc8ea4573451b759"); + installedFile.FileType = Manifest::InstalledFileTypeEnum::Launch; + installedFile.InvocationParameter = "invocation"; + installedFile.DisplayName = "name"; + installedFiles.InstalledFiles.Files.emplace_back(std::move(installedFile)); + Correlation::InstalledStartupLinkFile startupLink; + startupLink.RelativeFilePath = "TestApp.lnk"; + startupLink.FileType = Manifest::InstalledFileTypeEnum::Launch; + installedFiles.StartupLinkFiles.emplace_back(std::move(startupLink)); + + installedFilesData->InstallationMetadata = std::move(installedFiles); + + InstallerMetadataCollectionContext context = CreateTestContext(std::move(correlationData), std::move(installedFilesData), input); + TestOutput output = GetOutput(context); + + REQUIRE(output.IsSuccess()); + output.ValidateFieldPresence(); + + REQUIRE(output.Metadata->ProductVersionMin.ToString() == output.Metadata->ProductVersionMax.ToString()); + REQUIRE(output.Metadata->ProductVersionMin.ToString() == manifest.Version); + REQUIRE(output.Metadata->InstallerMetadataMap.size() == 1); + REQUIRE(output.Metadata->InstallerMetadataMap.count(input.InstallerHash.value()) == 1); + const auto& entry = output.Metadata->InstallerMetadataMap[input.InstallerHash.value()]; + REQUIRE(entry.SubmissionIdentifier == input.SubmissionIdentifier.value()); + REQUIRE(entry.Scope == metadata[PackageVersionMetadata::InstalledScope]); + REQUIRE(entry.AppsAndFeaturesEntries.size() == 1); + REQUIRE(entry.AppsAndFeaturesEntries[0].DisplayName == manifest.DefaultLocalization.Get<Manifest::Localization::PackageName>()); + REQUIRE(entry.AppsAndFeaturesEntries[0].Publisher == manifest.DefaultLocalization.Get<Manifest::Localization::Publisher>()); + REQUIRE(entry.AppsAndFeaturesEntries[0].DisplayVersion == manifest.Version); + REQUIRE(entry.AppsAndFeaturesEntries[0].ProductCode == manifest.Installers[0].ProductCode); + REQUIRE(entry.AppsAndFeaturesEntries[0].InstallerType == Manifest::InstallerTypeEnum::Msi); + REQUIRE(entry.InstalledFiles.has_value()); + REQUIRE(entry.InstalledFiles->DefaultInstallLocation == "%TEMP%\\TestApp"); + REQUIRE(entry.InstalledFiles->Files.size() == 1); + REQUIRE(entry.InstalledFiles->Files[0].RelativeFilePath == "test.exe"); + REQUIRE(entry.InstalledFiles->Files[0].FileSha256 == Utility::SHA256::ConvertToBytes("d2a45116709136462ee7a1c42f0e75f0efa258fe959b1504dc8ea4573451b759")); + REQUIRE(entry.InstalledFiles->Files[0].FileType == Manifest::InstalledFileTypeEnum::Launch); + REQUIRE(entry.InstalledFiles->Files[0].InvocationParameter == "invocation"); + REQUIRE(entry.InstalledFiles->Files[0].DisplayName == "name"); + REQUIRE(entry.StartupLinkFiles.has_value()); + REQUIRE(entry.StartupLinkFiles->size() == 1); + REQUIRE(entry.StartupLinkFiles->at(0).RelativeFilePath == "TestApp.lnk"); + REQUIRE(entry.StartupLinkFiles->at(0).FileType == Manifest::InstalledFileTypeEnum::Launch); + REQUIRE(output.Metadata->HistoricalMetadataList.empty()); +} + +TEST_CASE("MetadataCollection_NewPackage_NoInstallationMetadata", "[metadata_collection]") +{ + TestInput input(MinimalDefaults); + input.SupportedMetadataVersion = "1.2"; + auto correlationData = std::make_unique<TestARPCorrelationData>(); + + Manifest::Manifest manifest; + manifest.DefaultLocalization.Add<Manifest::Localization::PackageName>("Test Package Name"); + manifest.DefaultLocalization.Add<Manifest::Localization::Publisher>("Test Publisher"); + manifest.Version = "1.2.3"; + manifest.Installers.push_back({}); + manifest.Installers[0].ProductCode = "{guid}"; + + IPackageVersion::Metadata metadata; + metadata[PackageVersionMetadata::InstalledType] = Manifest::InstallerTypeToString(Manifest::InstallerTypeEnum::Msi); + + correlationData->CorrelateForNewlyInstalledResult.Package = std::make_shared<TestPackageVersion>(manifest, metadata); + + InstallerMetadataCollectionContext context = CreateTestContext(std::move(correlationData), input); + TestOutput output = GetOutput(context); + + REQUIRE(output.IsSuccess()); + output.ValidateFieldPresence(); + + REQUIRE(output.Metadata->InstallerMetadataMap.size() == 1); + REQUIRE(output.Metadata->InstallerMetadataMap.count(input.InstallerHash.value()) == 1); + const auto& entry = output.Metadata->InstallerMetadataMap[input.InstallerHash.value()]; + REQUIRE(entry.Scope.empty()); + REQUIRE_FALSE(entry.InstalledFiles.has_value()); + REQUIRE_FALSE(entry.StartupLinkFiles.has_value()); +} + +TEST_CASE("MetadataCollection_SameSubmission_SameInstaller_InstallationMetadata", "[metadata_collection]") +{ + std::string version = "1.3.5"; + std::string productCode = "{guid}"; + Manifest::InstallerTypeEnum installerType = Manifest::InstallerTypeEnum::Msi; + + TestInput input(MinimalDefaults, version, productCode, installerType); + input.SupportedMetadataVersion = "1.2"; + input.CurrentMetadata->SchemaVersion = { "1.2" }; + + Manifest::InstallationMetadataInfo installedFiles; + installedFiles.DefaultInstallLocation = "%TEMP%\\TestApp"; + Manifest::InstalledFile installedFile; + installedFile.RelativeFilePath = "test.exe"; + installedFile.FileSha256 = Utility::SHA256::ConvertToBytes("d2a45116709136462ee7a1c42f0e75f0efa258fe959b1504dc8ea4573451b759"); + installedFile.FileType = Manifest::InstalledFileTypeEnum::Launch; + installedFile.InvocationParameter = "invocation"; + installedFile.DisplayName = "name"; + installedFiles.Files.emplace_back(std::move(installedFile)); + input.CurrentMetadata->InstallerMetadataMap.begin()->second.InstalledFiles = std::move(installedFiles); + + std::vector<InstalledStartupLinkFile> startupLinkFiles; + Correlation::InstalledStartupLinkFile startupLink; + startupLink.RelativeFilePath = "TestApp.lnk"; + startupLink.FileType = Manifest::InstalledFileTypeEnum::Launch; + startupLinkFiles.emplace_back(std::move(startupLink)); + input.CurrentMetadata->InstallerMetadataMap.begin()->second.StartupLinkFiles = std::move(startupLinkFiles); + + auto correlationData = std::make_unique<TestARPCorrelationData>(); + auto installedFilesData = std::make_unique<TestInstalledFilesCorrelation>(); + + Manifest::Manifest manifest; + manifest.DefaultLocalization.Add<Manifest::Localization::PackageName>("Different Language Name"); + // Same publisher + manifest.DefaultLocalization.Add<Manifest::Localization::Publisher>(input.CurrentMetadata->InstallerMetadataMap.begin()->second.AppsAndFeaturesEntries[0].Publisher); + manifest.Version = version; + manifest.Installers.push_back({}); + manifest.Installers[0].ProductCode = productCode; + + IPackageVersion::Metadata metadata; + metadata[PackageVersionMetadata::InstalledType] = Manifest::InstallerTypeToString(installerType); + + correlationData->CorrelateForNewlyInstalledResult.Package = std::make_shared<TestPackageVersion>(manifest, metadata); + + Correlation::InstallationMetadata newInstalledFiles; + newInstalledFiles.InstalledFiles.DefaultInstallLocation = "%TEMP%\\NewTestApp"; + Manifest::InstalledFile newInstalledFile; + newInstalledFile.RelativeFilePath = "test.exe"; + newInstalledFile.FileSha256 = Utility::SHA256::ConvertToBytes("d2a45116709136462ee7a1c42f0e75f0efa258fe959b1504dc8ea4573451b759"); + newInstalledFile.FileType = Manifest::InstalledFileTypeEnum::Launch; + newInstalledFile.InvocationParameter = "invocation"; + newInstalledFile.DisplayName = "name"; + newInstalledFiles.InstalledFiles.Files.emplace_back(std::move(newInstalledFile)); + Correlation::InstalledStartupLinkFile newStartupLink; + newStartupLink.RelativeFilePath = "NewTestApp.lnk"; + newStartupLink.FileType = Manifest::InstalledFileTypeEnum::Launch; + newInstalledFiles.StartupLinkFiles.emplace_back(std::move(newStartupLink)); + + installedFilesData->InstallationMetadata = std::move(newInstalledFiles); + + InstallerMetadataCollectionContext context = CreateTestContext(std::move(correlationData), std::move(installedFilesData), input); + TestOutput output = GetOutput(context); + + REQUIRE(output.IsSuccess()); + output.ValidateFieldPresence(); + + REQUIRE(output.Metadata->InstallerMetadataMap.size() == 1); + REQUIRE(output.Metadata->InstallerMetadataMap.count(input.InstallerHash.value()) == 1); + const auto& entry = output.Metadata->InstallerMetadataMap[input.InstallerHash.value()]; + + // Conflicting installed files entries get removed. + REQUIRE(entry.InstalledFiles.has_value()); + REQUIRE_FALSE(entry.InstalledFiles->HasData()); + // Non duplicate startup links get added. + REQUIRE(entry.StartupLinkFiles.has_value()); + REQUIRE(entry.StartupLinkFiles->size() == 2); +} + +TEST_CASE("MetadataCollection_Merge_SameInstaller_InstalledFiles", "[metadata_collection]") +{ + TestMerge mergeData{ MinimalDefaults }; + mergeData.Metadatas->emplace_back(MakeProductMetadata()); + + Manifest::InstallationMetadataInfo installedFiles; + installedFiles.DefaultInstallLocation = "%TEMP%\\TestApp"; + Manifest::InstalledFile installedFile; + installedFile.RelativeFilePath = "test.exe"; + installedFile.FileSha256 = Utility::SHA256::ConvertToBytes("d2a45116709136462ee7a1c42f0e75f0efa258fe959b1504dc8ea4573451b759"); + installedFile.FileType = Manifest::InstalledFileTypeEnum::Launch; + installedFile.InvocationParameter = "invocation"; + installedFile.DisplayName = "name"; + installedFiles.Files.emplace_back(std::move(installedFile)); + + mergeData.Metadatas->at(0).SchemaVersion = { "1.2" }; + mergeData.Metadatas->at(0).InstallerMetadataMap.begin()->second.InstalledFiles = installedFiles; + mergeData.Metadatas->at(1).SchemaVersion = { "1.2" }; + + // Different default install location clears whole data + Manifest::InstallationMetadataInfo newInstalledFiles = installedFiles; + newInstalledFiles.DefaultInstallLocation = "%TEMP%\\NewTestApp"; + mergeData.Metadatas->at(1).InstallerMetadataMap.begin()->second.InstalledFiles = newInstalledFiles; + std::wstring mergeResult = InstallerMetadataCollectionContext::Merge(mergeData.ToJSON(), 0, {}); + REQUIRE(!mergeResult.empty()); + + ProductMetadata mergeMetadata; + mergeMetadata.FromJson(web::json::value::parse(mergeResult)); + + REQUIRE(mergeMetadata.InstallerMetadataMap.size() == 1); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.InstalledFiles.has_value()); + REQUIRE_FALSE(mergeMetadata.InstallerMetadataMap.begin()->second.InstalledFiles->HasData()); + + // Different RelativeFilePath clears the file entry + Manifest::InstallationMetadataInfo newInstalledFiles2 = installedFiles; + newInstalledFiles2.Files[0].RelativeFilePath = "test2.exe"; + mergeData.Metadatas->at(1).InstallerMetadataMap.begin()->second.InstalledFiles = newInstalledFiles2; + mergeResult = InstallerMetadataCollectionContext::Merge(mergeData.ToJSON(), 0, {}); + REQUIRE(!mergeResult.empty()); + + mergeMetadata.FromJson(web::json::value::parse(mergeResult)); + + REQUIRE(mergeMetadata.InstallerMetadataMap.size() == 1); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.InstalledFiles.has_value()); + REQUIRE_FALSE(mergeMetadata.InstallerMetadataMap.begin()->second.InstalledFiles->DefaultInstallLocation.empty()); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.InstalledFiles->Files.empty()); + + // Different other fields clears the fields themselves + Manifest::InstallationMetadataInfo newInstalledFiles3 = installedFiles; + newInstalledFiles3.Files[0].DisplayName = "name2"; + newInstalledFiles3.Files[0].FileSha256 = Utility::SHA256::ConvertToBytes("d2a45116709136462ee7a1c42f0e75f0efa258fe959b1504dc8ea4573451b756"); + newInstalledFiles3.Files[0].InvocationParameter = "invocation2"; + newInstalledFiles3.Files[0].FileType = Manifest::InstalledFileTypeEnum::Uninstall; + mergeData.Metadatas->at(1).InstallerMetadataMap.begin()->second.InstalledFiles = newInstalledFiles3; + mergeResult = InstallerMetadataCollectionContext::Merge(mergeData.ToJSON(), 0, {}); + REQUIRE(!mergeResult.empty()); + + mergeMetadata.FromJson(web::json::value::parse(mergeResult)); + + REQUIRE(mergeMetadata.InstallerMetadataMap.size() == 1); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.InstalledFiles.has_value()); + REQUIRE_FALSE(mergeMetadata.InstallerMetadataMap.begin()->second.InstalledFiles->DefaultInstallLocation.empty()); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.InstalledFiles->Files.size() == 1); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.InstalledFiles->Files[0].RelativeFilePath == "test.exe"); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.InstalledFiles->Files[0].DisplayName == ""); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.InstalledFiles->Files[0].InvocationParameter == ""); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.InstalledFiles->Files[0].FileType == Manifest::InstalledFileTypeEnum::Unknown); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.InstalledFiles->Files[0].FileSha256.empty()); +} + +TEST_CASE("MetadataCollection_Merge_SameInstaller_StartupLinkFiles", "[metadata_collection]") +{ + TestMerge mergeData{ MinimalDefaults }; + mergeData.Metadatas->emplace_back(MakeProductMetadata()); + + std::vector<InstalledStartupLinkFile> startupLinkFiles; + Correlation::InstalledStartupLinkFile startupLink; + startupLink.RelativeFilePath = "TestApp.lnk"; + startupLink.FileType = Manifest::InstalledFileTypeEnum::Launch; + startupLinkFiles.emplace_back(std::move(startupLink)); + + mergeData.Metadatas->at(0).SchemaVersion = { "1.2" }; + mergeData.Metadatas->at(0).InstallerMetadataMap.begin()->second.StartupLinkFiles = startupLinkFiles; + mergeData.Metadatas->at(1).SchemaVersion = { "1.2" }; + + // Different relative file path gets added + std::vector<InstalledStartupLinkFile> newStartupLinkFiles = startupLinkFiles; + newStartupLinkFiles[0].RelativeFilePath = "TestApp2.lnk"; + mergeData.Metadatas->at(1).InstallerMetadataMap.begin()->second.StartupLinkFiles = newStartupLinkFiles; + std::wstring mergeResult = InstallerMetadataCollectionContext::Merge(mergeData.ToJSON(), 0, {}); + REQUIRE(!mergeResult.empty()); + + ProductMetadata mergeMetadata; + mergeMetadata.FromJson(web::json::value::parse(mergeResult)); + + REQUIRE(mergeMetadata.InstallerMetadataMap.size() == 1); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.StartupLinkFiles.has_value()); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.StartupLinkFiles->size() == 2); + + // Different other fields clears the fields themselves + std::vector<InstalledStartupLinkFile> newStartupLinkFiles2 = startupLinkFiles; + newStartupLinkFiles2[0].FileType = Manifest::InstalledFileTypeEnum::Uninstall; + mergeData.Metadatas->at(1).InstallerMetadataMap.begin()->second.StartupLinkFiles = newStartupLinkFiles2; + mergeResult = InstallerMetadataCollectionContext::Merge(mergeData.ToJSON(), 0, {}); + REQUIRE(!mergeResult.empty()); + + mergeMetadata.FromJson(web::json::value::parse(mergeResult)); + + REQUIRE(mergeMetadata.InstallerMetadataMap.size() == 1); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.StartupLinkFiles.has_value()); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.StartupLinkFiles->size() == 1); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.StartupLinkFiles->at(0).RelativeFilePath == "TestApp.lnk"); + REQUIRE(mergeMetadata.InstallerMetadataMap.begin()->second.StartupLinkFiles->at(0).FileType == Manifest::InstalledFileTypeEnum::Unknown); +}+ \ No newline at end of file diff --git a/src/AppInstallerCommonCore/Filesystem.cpp b/src/AppInstallerCommonCore/Filesystem.cpp @@ -194,22 +194,22 @@ namespace AppInstaller::Filesystem try { - return Utility::ExpandEnvironmentVariables(Utility::ConvertToUTF16(trimPath)); + return std::filesystem::weakly_canonical(Utility::ExpandEnvironmentVariables(Utility::ConvertToUTF16(trimPath))); } catch (...) { - return path; + return Utility::ConvertToUTF16(path); } } - void ReplaceCommonPathPrefix(std::filesystem::path& source, const std::filesystem::path& prefix, std::string_view replacement) + bool ReplaceCommonPathPrefix(std::filesystem::path& source, const std::filesystem::path& prefix, std::string_view replacement) { auto prefixItr = prefix.begin(); auto sourceItr = source.begin(); while (prefixItr != prefix.end() && sourceItr != source.end()) { - if (*prefixItr != *sourceItr) + if (Utility::ICUCaseInsensitiveEquals(prefixItr->u8string(), sourceItr->u8string())) { break; } @@ -229,7 +229,11 @@ namespace AppInstaller::Filesystem } source = std::move(temp); + + return true; } + + return false; } std::filesystem::path GetKnownFolderPath(const KNOWNFOLDERID& id) diff --git a/src/AppInstallerCommonCore/FolderFileWatcher.cpp b/src/AppInstallerCommonCore/FolderFileWatcher.cpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include "winget/FolderFileWatcher.h" +#include "AppInstallerStrings.h" namespace AppInstaller::Utility { @@ -26,8 +27,7 @@ namespace AppInstaller::Utility case wil::FolderChangeEvent::RenameNewName: { std::filesystem::path path(filePath); - if (!m_ext.has_value() || - (m_ext.has_value() && path.extension() == m_ext)) + if (!m_ext.has_value() || Utility::CaseInsensitiveEquals(path.extension().u8string(), *m_ext)) { m_files.emplace(path); } @@ -40,8 +40,7 @@ namespace AppInstaller::Utility case wil::FolderChangeEvent::RenameOldName: { std::filesystem::path path(filePath); - if (!m_ext.has_value() || - (m_ext.has_value() && path.extension() == m_ext)) + if (!m_ext.has_value() || Utility::CaseInsensitiveEquals(path.extension().u8string(), *m_ext)) { auto it = m_files.find(path); if (it != m_files.cend()) diff --git a/src/AppInstallerCommonCore/Manifest/ManifestCommon.cpp b/src/AppInstallerCommonCore/Manifest/ManifestCommon.cpp @@ -450,6 +450,21 @@ namespace AppInstaller::Manifest return "Unknown"sv; } + std::string_view InstalledFileTypeToString(InstalledFileTypeEnum installedFileType) + { + switch (installedFileType) + { + case InstalledFileTypeEnum::Launch: + return "launch"sv; + case InstalledFileTypeEnum::Uninstall: + return "uninstall"sv; + case InstalledFileTypeEnum::Other: + return "other"sv; + } + + return "unknown"; + } + bool DoesInstallerTypeUsePackageFamilyName(InstallerTypeEnum installerType) { return (installerType == InstallerTypeEnum::Msix || installerType == InstallerTypeEnum::MSStore); diff --git a/src/AppInstallerCommonCore/Public/winget/Filesystem.h b/src/AppInstallerCommonCore/Public/winget/Filesystem.h @@ -38,7 +38,8 @@ namespace AppInstaller::Filesystem std::filesystem::path GetExpandedPath(const std::string& path); // If `source` begins with all of `prefix`, replace that with `replacement`. - void ReplaceCommonPathPrefix(std::filesystem::path& source, const std::filesystem::path& prefix, std::string_view replacement); + // Returns true if replacement happened, false otherwise. + bool ReplaceCommonPathPrefix(std::filesystem::path& source, const std::filesystem::path& prefix, std::string_view replacement); // Gets the path of a known folder. std::filesystem::path GetKnownFolderPath(const KNOWNFOLDERID& id); diff --git a/src/AppInstallerCommonCore/Public/winget/FolderFileWatcher.h b/src/AppInstallerCommonCore/Public/winget/FolderFileWatcher.h @@ -1,7 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. #pragma once -#include "pch.h" +#include <unordered_set> +#pragma warning( push ) +#pragma warning ( disable : 6387 28196 ) +#include <wil/filesystem.h> +#pragma warning( pop ) namespace AppInstaller::Utility { @@ -14,13 +18,14 @@ namespace AppInstaller::Utility FolderFileWatcher(const FolderFileWatcher&) = delete; FolderFileWatcher& operator=(const FolderFileWatcher&) = delete; - FolderFileWatcher(FolderFileWatcher&&) = delete; - FolderFileWatcher& operator=(FolderFileWatcher&&) = delete; + FolderFileWatcher(FolderFileWatcher&&) = default; + FolderFileWatcher& operator=(FolderFileWatcher&&) = default; void Start(); void Stop(); - const std::unordered_set<std::filesystem::path>& files() { return m_files; } + const std::unordered_set<std::filesystem::path>& Files() { return m_files; } + const std::filesystem::path& FolderPath() { return m_path; } private: std::filesystem::path m_path; diff --git a/src/AppInstallerCommonCore/Public/winget/ManifestCommon.h b/src/AppInstallerCommonCore/Public/winget/ManifestCommon.h @@ -292,6 +292,8 @@ namespace AppInstaller::Manifest // Checks if there are any installation metadata available. bool HasData() const { return !DefaultInstallLocation.empty() || !Files.empty(); } + + void Clear() { DefaultInstallLocation.clear(); Files.clear(); } }; InstallerTypeEnum ConvertToInstallerTypeEnum(const std::string& in); @@ -318,6 +320,8 @@ namespace AppInstaller::Manifest std::string_view ScopeToString(ScopeEnum scope); + std::string_view InstalledFileTypeToString(InstalledFileTypeEnum installedFileType); + // Gets a value indicating whether the given installer uses the PackageFamilyName system reference. bool DoesInstallerTypeUsePackageFamilyName(InstallerTypeEnum installerType); diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -291,6 +291,7 @@ <ClInclude Include="PackageTrackingCatalogSourceFactory.h" /> <ClInclude Include="pch.h" /> <ClInclude Include="Public\winget\ARPCorrelation.h" /> + <ClInclude Include="Public\winget\InstalledFilesCorrelation.h" /> <ClInclude Include="Public\winget\InstallerMetadataCollectionContext.h" /> <ClInclude Include="Public\winget\ManifestJSONParser.h" /> <ClInclude Include="Public\winget\ARPCorrelationAlgorithms.h" /> @@ -339,6 +340,7 @@ <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">NotUsing</PrecompiledHeader> <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">NotUsing</PrecompiledHeader> </ClCompile> + <ClCompile Include="InstalledFilesCorrelation.cpp" /> <ClCompile Include="InstallerMetadataCollectionContext.cpp" /> <ClCompile Include="ManifestJSONParser.cpp" /> <ClCompile Include="Microsoft\ARPHelper.cpp" /> diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -354,6 +354,9 @@ <ClInclude Include="Microsoft\Schema\Pinning_1_0\PinTable.h"> <Filter>Microsoft\Schema\Pinning_1_0</Filter> </ClInclude> + <ClInclude Include="Public\winget\InstalledFilesCorrelation.h"> + <Filter>Public\winget</Filter> + </ClInclude> </ItemGroup> <ItemGroup> <ClCompile Include="pch.cpp"> @@ -563,6 +566,9 @@ <ClCompile Include="Microsoft\Schema\Pinning_1_0\PinTable.cpp"> <Filter>Microsoft\Schema\Pinning_1_0</Filter> </ClCompile> + <ClCompile Include="InstalledFilesCorrelation.cpp"> + <Filter>Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <None Include="PropertySheet.props" /> diff --git a/src/AppInstallerRepositoryCore/InstalledFilesCorrelation.cpp b/src/AppInstallerRepositoryCore/InstalledFilesCorrelation.cpp @@ -0,0 +1,322 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "winget/InstalledFilesCorrelation.h" +#include <winget/FolderFileWatcher.h> +#include <winget/Filesystem.h> + +using namespace AppInstaller::Manifest; +using namespace AppInstaller::Repository; +using namespace AppInstaller::Utility; + +namespace AppInstaller::Repository::Correlation +{ + namespace + { + constexpr std::string_view s_ShellLinkFileExtension = ".lnk"sv; + const std::vector<std::pair<std::filesystem::path, std::string>> s_CandidateInstallLocationRoots = + { + { Filesystem::GetKnownFolderPath(FOLDERID_LocalAppData), "%LOCALAPPDATA%"}, + { Filesystem::GetKnownFolderPath(FOLDERID_ProgramFiles), "%PROGRAMFILES%" }, + { Filesystem::GetKnownFolderPath(FOLDERID_ProgramFilesX86), "%PROGRAMFILES(X86)%" }, + }; + + // Contains shell link info + struct ShellLinkFileInfo + { + std::filesystem::path Path; + std::string Args; + std::string DisplayName; + }; + + std::optional<ShellLinkFileInfo> ParseShellLinkFile(const std::filesystem::path& linkFile) + { + try + { + AICLI_LOG(Repo, Info, << "Parsing link file at " << linkFile); + + ShellLinkFileInfo result; + + Microsoft::WRL::ComPtr<IShellLink> shellLink; + THROW_IF_FAILED(CoCreateInstance(CLSID_ShellLink, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&shellLink))); + Microsoft::WRL::ComPtr<IPersistFile> persistFile; + THROW_IF_FAILED(shellLink.As(&persistFile)); + THROW_IF_FAILED(persistFile->Load(linkFile.wstring().c_str(), STGM_READ)); + THROW_IF_FAILED(shellLink->Resolve(nullptr, SLR_NO_UI | SLR_NOUPDATE | SLR_NOSEARCH | SLR_NOTRACK | SLR_NOLINKINFO)); + + { + // Parse Path from shell link + std::wstring buffer; + buffer.resize(MAX_PATH); + HRESULT hr = S_OK; + for (int retry = 0; retry < 5; retry++) + { + hr = shellLink->GetPath( + &buffer[0], + static_cast<int>(buffer.size()), + nullptr, + 0 + ); + + if (SUCCEEDED(hr)) + { + buffer.erase(std::find(buffer.begin(), buffer.end(), L'\0'), buffer.end()); + result.Path = buffer; + break; + } + else if (hr == HRESULT_FROM_WIN32(ERROR_INSUFFICIENT_BUFFER)) + { + buffer.resize(buffer.size() * 2); + } + else + { + THROW_IF_FAILED(hr); + } + } + } + + { + // Parse arguments from shell link + std::wstring buffer; + buffer.resize(MAX_PATH); + HRESULT hr = S_OK; + for (int retry = 0; retry < 5; retry++) + { + hr = shellLink->GetArguments( + &buffer[0], + static_cast<int>(buffer.size())); + + if (SUCCEEDED(hr)) + { + buffer.erase(std::find(buffer.begin(), buffer.end(), L'\0'), buffer.end()); + result.Args = Utility::ConvertToUTF8(buffer); + break; + } + else if (hr == HRESULT_FROM_WIN32(ERROR_INSUFFICIENT_BUFFER)) + { + buffer.resize(buffer.size() * 2); + } + else + { + THROW_IF_FAILED(hr); + } + } + } + + // Use shell link file name (minus extension) as display name. + result.DisplayName = linkFile.stem().u8string(); + + AICLI_LOG(Repo, Info, << "Link file parsed. Path: " << result.Path << " Args: " << result.Args << " DisplayName: " << result.DisplayName); + + return result; + } + catch (...) + { + AICLI_LOG(Repo, Error, << "Failed to parse link file at " << linkFile); + return {}; + } + } + + // Returns nullopt if path is not under base. + std::optional<std::filesystem::path> GetRelativePath(const std::filesystem::path& path, const std::filesystem::path& base) + { + auto canonicalPath = std::filesystem::weakly_canonical(path); + auto canonicalBase = std::filesystem::weakly_canonical(base); + + auto relativePath = std::filesystem::relative(canonicalPath, canonicalBase); + if (!relativePath.empty() && *relativePath.begin() != "." && *relativePath.begin() != "..") + { + return relativePath; + } + else + { + return {}; + } + } + + std::optional<std::filesystem::path> CheckOneInstallLocation(const std::filesystem::path& childFile, const std::filesystem::path& baseFolder) + { + auto relativePath = GetRelativePath(childFile, baseFolder); + if (relativePath) + { + // TODO: Here we assume the install location is the top directory of relative path. + auto installLocation = baseFolder / *relativePath->begin(); + if (std::filesystem::exists(installLocation) && std::filesystem::is_directory(installLocation)) + { + return installLocation; + } + } + + return {}; + } + + // If install location is not provided in arp entry, try LocalAppData folder and Program Files folders. + std::optional<std::filesystem::path> CheckInstallLocation(const std::filesystem::path& path) + { + for (auto const& entry : s_CandidateInstallLocationRoots) + { + auto installLocation = CheckOneInstallLocation(path, entry.first); + if (installLocation) + { + return installLocation; + } + } + + return {}; + } + + // TODO: basic heuristics to determine file type. + AppInstaller::Manifest::InstalledFileTypeEnum GetInstalledFileType(const ShellLinkFileInfo& linkInfo) + { + Manifest::InstalledFileTypeEnum result = Manifest::InstalledFileTypeEnum::Other; + + if (Utility::CaseInsensitiveContainsSubstring(linkInfo.Path.u8string(), "uninstall") || + Utility::CaseInsensitiveContainsSubstring(linkInfo.Path.u8string(), "unins000") || + Utility::CaseInsensitiveContainsSubstring(linkInfo.Args, "uninstall") || + Utility::CaseInsensitiveContainsSubstring(linkInfo.DisplayName, "uninstall")) + { + result = Manifest::InstalledFileTypeEnum::Uninstall; + } + else if (Utility::CaseInsensitiveEquals(linkInfo.Path.extension().u8string(), ".exe")) + { + result = Manifest::InstalledFileTypeEnum::Launch; + } + + return result; + } + + std::string GetUnexpandedInstallLocation(const std::filesystem::path& installLocation) + { + // Try to match the candidate install location roots first. + std::filesystem::path resultInstallLocation = installLocation; + for (auto const& entry : s_CandidateInstallLocationRoots) + { + if (Filesystem::ReplaceCommonPathPrefix(resultInstallLocation, entry.first, entry.second)) + { + return resultInstallLocation.u8string(); + } + } + + // Then try PathUnExpandEnvStrings OS api + std::wstring installLocationWString = installLocation.wstring(); + std::wstring buffer; + buffer.resize(installLocationWString.size() + 20); + if (PathUnExpandEnvStrings( + installLocationWString.c_str(), + &buffer[0], + static_cast<int>(buffer.size()))) + { + buffer.resize(buffer.find(L'\0')); + return Utility::ConvertToUTF8(buffer); + } + + return resultInstallLocation.u8string(); + } + } + + InstalledFilesCorrelation::InstalledFilesCorrelation() + { + m_fileWatchers.emplace_back(Filesystem::GetKnownFolderPath(FOLDERID_CommonStartMenu), std::string{ s_ShellLinkFileExtension }); + m_fileWatchers.emplace_back(Filesystem::GetKnownFolderPath(FOLDERID_StartMenu), std::string{ s_ShellLinkFileExtension }); + } + + void InstalledFilesCorrelation::StartFileWatcher() + { + m_files.clear(); + + for (auto& watcher : m_fileWatchers) + { + watcher.Start(); + } + } + + void InstalledFilesCorrelation::StopFileWatcher() + { + for (auto& watcher : m_fileWatchers) + { + watcher.Stop(); + } + + for (auto& watcher : m_fileWatchers) + { + FileWatcherFiles files; + files.Folder = watcher.FolderPath(); + + for (auto const& file : watcher.Files()) + { + files.Files.emplace_back(file); + } + + m_files.emplace_back(std::move(files)); + } + } + + InstallationMetadata InstalledFilesCorrelation::CorrelateForNewlyInstalled( + const Manifest::Manifest&, + const std::string& arpInstallLocation) + { + InstallationMetadata result; + + std::filesystem::path installLocation; + // Use arp install location if provided + if (!arpInstallLocation.empty()) + { + installLocation = Filesystem::GetExpandedPath(arpInstallLocation); + } + + for (auto const& files : m_files) + { + for (auto const& file : files.Files) + { + // TODO: we only watch shell link files at the moment. + auto linkInfo = ParseShellLinkFile(files.Folder / file); + if (linkInfo) + { + auto installedFileType = GetInstalledFileType(linkInfo.value()); + + // Collect installed files metadata if exist + if (std::filesystem::exists(linkInfo->Path) && std::filesystem::is_regular_file(linkInfo->Path)) + { + if (installLocation.empty()) + { + // TODO: In most cases, installed files are under same folder, so use the first file to determine install location at the moment. + auto location = CheckInstallLocation(linkInfo->Path); + if (!location) + { + continue; + } + + installLocation = location.value(); + } + + auto relativePath = GetRelativePath(linkInfo->Path, installLocation); + if (relativePath) + { + AppInstaller::Manifest::InstalledFile fileEntry; + fileEntry.RelativeFilePath = relativePath->string(); + std::ifstream in{ linkInfo->Path, std::ifstream::binary }; + fileEntry.FileSha256 = Utility::SHA256::ComputeHash(in); + fileEntry.InvocationParameter = linkInfo->Args; + fileEntry.DisplayName = linkInfo->DisplayName; + fileEntry.FileType = installedFileType; + result.InstalledFiles.Files.emplace_back(std::move(fileEntry)); + } + } + + // Collect short cut paths + InstalledStartupLinkFile linkFile; + linkFile.RelativeFilePath = file.u8string(); + linkFile.FileType = installedFileType; + result.StartupLinkFiles.emplace_back(linkFile); + } + } + } + + if (!installLocation.empty()) + { + result.InstalledFiles.DefaultInstallLocation = GetUnexpandedInstallLocation(installLocation); + } + + return result; + } +} diff --git a/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp b/src/AppInstallerRepositoryCore/InstallerMetadataCollectionContext.cpp @@ -9,7 +9,7 @@ #include <AppInstallerFileLogger.h> #include <winget/TraceLogger.h> #include <AppInstallerStrings.h> - +#include <winget/JsonUtil.h> #include <winget/ManifestJSONParser.h> using namespace AppInstaller::Utility; @@ -27,6 +27,22 @@ namespace AppInstaller::Repository::Metadata SchemaVersion = L"1.1"; Scope = L"scope"; } + + if (::AppInstaller::Utility::Version{ "1.2" } <= version) + { + SchemaVersion = L"1.2"; + InstalledFiles = L"installedFiles"; + DefaultInstallLocation = L"DefaultInstallLocation"; + InstallationMetadataFiles = L"Files"; + InstalledFileRelativeFilePath = L"RelativeFilePath"; + InstalledFileSha256 = L"FileSha256"; + InstalledFileType = L"FileType"; + InstalledFileInvocationParameter = L"InvocationParameter"; + InstalledFileDisplayName = L"DisplayName"; + InstalledStartupLinks = L"startupLinks"; + InstalledStartupLinkPath = L"RelativeFilePath"; + InstalledStartupLinkType = L"FileType"; + } } utility::string_t SchemaVersion = L"1.0"; @@ -41,6 +57,7 @@ namespace AppInstaller::Repository::Metadata utility::string_t AppsAndFeaturesEntries = L"AppsAndFeaturesEntries"; utility::string_t Historical = L"historical"; + // AppsAndFeaturesEntry fields. utility::string_t DisplayName = L"DisplayName"; utility::string_t Publisher = L"Publisher"; utility::string_t DisplayVersion = L"DisplayVersion"; @@ -57,6 +74,22 @@ namespace AppInstaller::Repository::Metadata // 1.1 utility::string_t Scope; + + // 1.2 + + // Installed files + utility::string_t InstalledFiles; + utility::string_t DefaultInstallLocation; + utility::string_t InstallationMetadataFiles; + utility::string_t InstalledFileRelativeFilePath; + utility::string_t InstalledFileSha256; + utility::string_t InstalledFileType; + utility::string_t InstalledFileInvocationParameter; + utility::string_t InstalledFileDisplayName; + // Startup links + utility::string_t InstalledStartupLinks; + utility::string_t InstalledStartupLinkPath; + utility::string_t InstalledStartupLinkType; }; struct OutputFields_1_0 @@ -189,6 +222,110 @@ namespace AppInstaller::Repository::Metadata json[field] = AppInstaller::JSON::GetStringValue(value); } } + + // For installed files merging, we remove conflicting entries, like scope. Indicating we are not certain some files will always be installed. + void MergeInstalledFilesMetadata(Manifest::InstallationMetadataInfo& existing, const Manifest::InstallationMetadataInfo& incoming) + { + if (!Utility::CaseInsensitiveEquals(existing.DefaultInstallLocation, incoming.DefaultInstallLocation)) + { + existing.Clear(); + return; + } + + auto existingItr = existing.Files.begin(); + while (existingItr != existing.Files.end()) + { + auto itr = std::find_if(incoming.Files.begin(), incoming.Files.end(), [&](const Manifest::InstalledFile& entry) + { + return Utility::CaseInsensitiveEquals(existingItr->RelativeFilePath, entry.RelativeFilePath); + }); + + if (itr == incoming.Files.end()) + { + existingItr = existing.Files.erase(existingItr); + } + else + { + if (existingItr->InvocationParameter != itr->InvocationParameter) + { + existingItr->InvocationParameter.clear(); + } + if (!Utility::CaseInsensitiveEquals(existingItr->DisplayName, itr->DisplayName)) + { + existingItr->DisplayName.clear(); + } + if (!Utility::SHA256::AreEqual(existingItr->FileSha256, itr->FileSha256)) + { + existingItr->FileSha256.clear(); + } + if (existingItr->FileType != itr->FileType) + { + existingItr->FileType = Manifest::InstalledFileTypeEnum::Unknown; + } + + ++existingItr; + } + } + } + + // For startup link files merging, we add non duplicate entries, like ProductCodes. Indicating possible startup links an installer could potentially add. + void MergeStartupLinkFilesMetadata(std::vector<Correlation::InstalledStartupLinkFile>& existing, const std::vector<Correlation::InstalledStartupLinkFile>& incoming) + { + for (auto const& incomingEntry : incoming) + { + auto itr = std::find_if(existing.begin(), existing.end(), [&](const Correlation::InstalledStartupLinkFile& entry) + { + return Utility::CaseInsensitiveEquals(incomingEntry.RelativeFilePath, entry.RelativeFilePath); + }); + + if (itr == existing.end()) + { + existing.emplace_back(incomingEntry); + } + else if (itr->FileType != incomingEntry.FileType) + { + // Set conflicting file type to Unknown. + itr->FileType = AppInstaller::Manifest::InstalledFileTypeEnum::Unknown; + } + } + } + + // TODO: This method could be moved to rest response parser and reused when winget supports launch + // scenarios (i.e. when startup links info are exposed in winget manifest). + std::optional<std::vector<Correlation::InstalledStartupLinkFile>> DeserializeInstalledStartupLinks( + const web::json::value& startupLinkFiles, + const ProductMetadataFields_1_N& fields) + { + if (startupLinkFiles.is_null() || !startupLinkFiles.is_array()) + { + return {}; + } + + std::vector<Correlation::InstalledStartupLinkFile> startupLinks; + for (auto const& startupLink : startupLinkFiles.as_array()) + { + Correlation::InstalledStartupLinkFile fileEntry; + + std::optional<std::string> relativeFilePath = AppInstaller::JSON::GetRawStringValueFromJsonNode(startupLink, fields.InstalledStartupLinkPath); + if (!AppInstaller::JSON::IsValidNonEmptyStringValue(relativeFilePath)) + { + AICLI_LOG(Repo, Error, << "Missing RelativeFilePath in Installed Startup Link Files."); + return {}; + } + + fileEntry.RelativeFilePath = std::move(*relativeFilePath); + + std::optional<std::string> fileType = AppInstaller::JSON::GetRawStringValueFromJsonNode(startupLink, fields.InstalledStartupLinkType); + if (AppInstaller::JSON::IsValidNonEmptyStringValue(fileType)) + { + fileEntry.FileType = Manifest::ConvertToInstalledFileTypeEnum(*fileType); + } + + startupLinks.emplace_back(std::move(fileEntry)); + } + + return startupLinks; + } } void ProductMetadata::Clear() @@ -347,8 +484,8 @@ namespace AppInstaller::Repository::Metadata ProductVersionMax = Version{ std::move(productVersionMaxString).value() }; } - // The 1.0 version of metadata uses the 1.1 version of REST - JSON::ManifestJSONParser parser{ Version{ "1.1" } }; + // The 1.0 version of metadata uses the 1.4 version of REST + JSON::ManifestJSONParser parser{ Version{ "1.4" } }; std::string submissionIdentifierVerification; @@ -380,6 +517,24 @@ namespace AppInstaller::Repository::Metadata installerMetadata.Scope = GetStringFromFutureSchema(item, fields.Scope).value_or(std::string{}); + if (!fields.InstalledFiles.empty()) + { + auto installedFiles = AppInstaller::JSON::GetJsonValueFromNode(item, fields.InstalledFiles); + if (installedFiles) + { + installerMetadata.InstalledFiles = parser.DeserializeInstallationMetadata(installedFiles->get()); + } + } + + if (!fields.InstalledStartupLinks.empty()) + { + auto startupLinks = AppInstaller::JSON::GetJsonValueFromNode(item, fields.InstalledStartupLinks); + if (startupLinks) + { + installerMetadata.StartupLinkFiles = DeserializeInstalledStartupLinks(startupLinks->get(), fields); + } + } + InstallerMetadataMap[installerHashString] = std::move(installerMetadata); } } @@ -424,6 +579,47 @@ namespace AppInstaller::Repository::Metadata itemValue[fields.InstallerHash] = AppInstaller::JSON::GetStringValue(item.first); itemValue[fields.SubmissionIdentifier] = AppInstaller::JSON::GetStringValue(item.second.SubmissionIdentifier); SetStringFromFutureSchema(itemValue, fields.Scope, item.second.Scope); + if (!fields.InstalledFiles.empty() && item.second.InstalledFiles.has_value()) + { + web::json::value installationMetadata; + + installationMetadata[fields.DefaultInstallLocation] = AppInstaller::JSON::GetStringValue(item.second.InstalledFiles->DefaultInstallLocation); + + web::json::value installedFilesArray = web::json::value::array(); + size_t installedFileIndex = 0; + for (const auto& entry : item.second.InstalledFiles->Files) + { + web::json::value entryValue; + AddFieldIfNotEmpty(entryValue, fields.InstalledFileRelativeFilePath, entry.RelativeFilePath); + AddFieldIfNotEmpty(entryValue, fields.InstalledFileInvocationParameter, entry.InvocationParameter); + AddFieldIfNotEmpty(entryValue, fields.InstalledFileDisplayName, entry.DisplayName); + entryValue[fields.InstalledFileType] = AppInstaller::JSON::GetStringValue(Manifest::InstalledFileTypeToString(entry.FileType)); + if (!entry.FileSha256.empty()) + { + entryValue[fields.InstalledFileSha256] = AppInstaller::JSON::GetStringValue(SHA256::ConvertToString(entry.FileSha256)); + } + installedFilesArray[installedFileIndex++] = std::move(entryValue); + } + installationMetadata[fields.InstallationMetadataFiles] = std::move(installedFilesArray); + + itemValue[fields.InstalledFiles] = std::move(installationMetadata); + } + + if (!fields.InstalledStartupLinks.empty() && item.second.StartupLinkFiles.has_value()) + { + web::json::value startupLinkFilesArray = web::json::value::array(); + size_t startupLinkFileIndex = 0; + for (const auto& entry : item.second.StartupLinkFiles.value()) + { + web::json::value entryValue; + entryValue[fields.InstalledStartupLinkPath] = AppInstaller::JSON::GetStringValue(entry.RelativeFilePath); + entryValue[fields.InstalledStartupLinkType] = AppInstaller::JSON::GetStringValue(Manifest::InstalledFileTypeToString(entry.FileType)); + + startupLinkFilesArray[startupLinkFileIndex++] = std::move(entryValue); + } + + itemValue[fields.InstalledStartupLinks] = std::move(startupLinkFilesArray); + } web::json::value appsAndFeaturesArray = web::json::value::array(); size_t appsAndFeaturesEntryIndex = 0; @@ -484,11 +680,15 @@ namespace AppInstaller::Repository::Metadata } InstallerMetadataCollectionContext::InstallerMetadataCollectionContext() : - m_correlationData(std::make_unique<Correlation::ARPCorrelationData>()) + m_correlationData(std::make_unique<Correlation::ARPCorrelationData>()), + m_installedFilesCorrelation(std::make_unique<Correlation::InstalledFilesCorrelation>()) {} - InstallerMetadataCollectionContext::InstallerMetadataCollectionContext(std::unique_ptr<Correlation::ARPCorrelationData> correlationData, const std::wstring& json) : - m_correlationData(std::move(correlationData)) + InstallerMetadataCollectionContext::InstallerMetadataCollectionContext( + std::unique_ptr<Correlation::ARPCorrelationData> correlationData, + std::unique_ptr<Correlation::InstalledFilesCorrelation> installedFilesCorrelation, + const std::wstring& json) : + m_correlationData(std::move(correlationData)), m_installedFilesCorrelation(std::move(installedFilesCorrelation)) { auto threadGlobalsLifetime = InitializeLogging({}); InitializePreinstallState(json); @@ -645,6 +845,7 @@ namespace AppInstaller::Repository::Metadata { // Collect post-install system state m_correlationData->CapturePostInstallSnapshot(); + m_installedFilesCorrelation->StopFileWatcher(); ComputeOutputData(); @@ -740,6 +941,7 @@ namespace AppInstaller::Repository::Metadata // Collect pre-install system state m_correlationData->CapturePreInstallSnapshot(); + m_installedFilesCorrelation->StartFileWatcher(); } catch (...) { @@ -753,14 +955,15 @@ namespace AppInstaller::Repository::Metadata m_outputMetadata.CopyFrom(m_currentMetadata, m_submissionIdentifier); Correlation::ARPCorrelationSettings settings; + std::string arpInstallLocation; // As this code is typically run in a controlled environment, we can assume that a single value change is very likely the correct value. settings.AllowSingleChange = true; + // ARP entry correlation Correlation::ARPCorrelationResult correlationResult = m_correlationData->CorrelateForNewlyInstalled(m_incomingManifest, settings); if (correlationResult.Package) { - m_outputStatus = OutputStatus::Success; auto& package = correlationResult.Package; // Update min and max versions based on the version of the correlated package @@ -780,6 +983,9 @@ namespace AppInstaller::Repository::Metadata Manifest::AppsAndFeaturesEntry newEntry; auto packageMetadata = package->GetMetadata(); + // Arp installed location will be used in later installed files correlation. + arpInstallLocation = packageMetadata[PackageVersionMetadata::InstalledLocation]; + // TODO: Use some amount of normalization here to prevent things like versions being in the name from bloating the data newEntry.DisplayName = package->GetProperty(PackageVersionProperty::Name).get(); newEntry.DisplayVersion = packageVersion.ToString(); @@ -831,6 +1037,66 @@ namespace AppInstaller::Repository::Metadata FilterAndAddToEntries(std::move(newEntry), itr->second.AppsAndFeaturesEntries); } } + + // Installation files correlation + auto installationMetadata = m_installedFilesCorrelation->CorrelateForNewlyInstalled(m_incomingManifest, arpInstallLocation); + + if (installationMetadata.InstalledFiles.HasData() || !installationMetadata.StartupLinkFiles.empty()) + { + // Add or update the metadata for the installer hash + auto itr = m_outputMetadata.InstallerMetadataMap.find(m_installerHash); + + if (itr == m_outputMetadata.InstallerMetadataMap.end()) + { + // New entry needed + ProductMetadata::InstallerMetadata newMetadata; + + newMetadata.SubmissionIdentifier = m_submissionIdentifier; + + if (installationMetadata.InstalledFiles.HasData()) + { + newMetadata.InstalledFiles = std::move(installationMetadata.InstalledFiles); + } + if (!installationMetadata.StartupLinkFiles.empty()) + { + newMetadata.StartupLinkFiles = std::move(installationMetadata.StartupLinkFiles); + } + + m_outputMetadata.InstallerMetadataMap[m_installerHash] = std::move(newMetadata); + } + else + { + // Add new or merge with existing entry + if (installationMetadata.InstalledFiles.HasData()) + { + if (!itr->second.InstalledFiles.has_value()) + { + itr->second.InstalledFiles = std::move(installationMetadata.InstalledFiles); + } + else + { + MergeInstalledFilesMetadata(*(itr->second.InstalledFiles), installationMetadata.InstalledFiles); + } + } + + if (!installationMetadata.StartupLinkFiles.empty()) + { + if (!itr->second.StartupLinkFiles.has_value()) + { + itr->second.StartupLinkFiles = std::move(installationMetadata.StartupLinkFiles); + } + else + { + MergeStartupLinkFilesMetadata(*(itr->second.StartupLinkFiles), installationMetadata.StartupLinkFiles); + } + } + } + } + + if (correlationResult.Package) + { + m_outputStatus = OutputStatus::Success; + } else { m_outputStatus = OutputStatus::LowConfidence; @@ -897,8 +1163,8 @@ namespace AppInstaller::Repository::Metadata m_installerHash = GetRequiredString(packageDataValue.value(), installerHashFieldName); - // The 1.0 version of input uses the 1.1 version of REST - JSON::ManifestJSONParser parser{ Version{ "1.1" }}; + // The 1.0 version of input uses the 1.4 version of REST + JSON::ManifestJSONParser parser{ Version{ "1.4" }}; { auto defaultLocaleValue = AppInstaller::JSON::GetJsonValueFromNode(packageDataValue.value(), defaultLocaleFieldName); @@ -1108,6 +1374,24 @@ namespace AppInstaller::Repository::Metadata } } + if (!itr->second.InstalledFiles.has_value()) + { + itr->second.InstalledFiles = installerMetadata.second.InstalledFiles; + } + else if (installerMetadata.second.InstalledFiles.has_value()) + { + MergeInstalledFilesMetadata(*(itr->second.InstalledFiles), *(installerMetadata.second.InstalledFiles)); + } + + if (!itr->second.StartupLinkFiles.has_value()) + { + itr->second.StartupLinkFiles = installerMetadata.second.StartupLinkFiles; + } + else if (installerMetadata.second.StartupLinkFiles.has_value()) + { + MergeStartupLinkFilesMetadata(*(itr->second.StartupLinkFiles), *(installerMetadata.second.StartupLinkFiles)); + } + // Merge into existing installer data for (const auto& targetEntry : installerMetadata.second.AppsAndFeaturesEntries) { diff --git a/src/AppInstallerRepositoryCore/PackageInstalledStatus.cpp b/src/AppInstallerRepositoryCore/PackageInstalledStatus.cpp @@ -175,7 +175,7 @@ namespace AppInstaller::Repository { installerStatus.Status.emplace_back( InstalledStatusType::AppsAndFeaturesEntryInstallLocation, - installedLocation.string(), + installedLocation.u8string(), installedLocationStatus); } @@ -186,12 +186,12 @@ namespace AppInstaller::Repository { for (auto const& file : installer.InstallationMetadata.Files) { - std::filesystem::path filePath = installedLocation / std::filesystem::path{ static_cast<std::string>(file.RelativeFilePath) }; + std::filesystem::path filePath = installedLocation / Utility::ConvertToUTF16(file.RelativeFilePath); auto fileStatus = CheckInstalledFileStatus(filePath, checkFileHash ? file.FileSha256 : Utility::SHA256::HashBuffer{}, fileHashes); installerStatus.Status.emplace_back( InstalledStatusType::AppsAndFeaturesEntryInstallLocationFile, - filePath.string(), + filePath.u8string(), fileStatus); } } @@ -208,7 +208,7 @@ namespace AppInstaller::Repository { installerStatus.Status.emplace_back( InstalledStatusType::DefaultInstallLocation, - defaultInstalledLocation.string(), + defaultInstalledLocation.u8string(), defaultInstalledLocationStatus); } @@ -218,12 +218,12 @@ namespace AppInstaller::Repository { for (auto const& file : installer.InstallationMetadata.Files) { - std::filesystem::path filePath = defaultInstalledLocation / std::filesystem::path{ static_cast<std::string>(file.RelativeFilePath) }; + std::filesystem::path filePath = defaultInstalledLocation / Utility::ConvertToUTF16(file.RelativeFilePath); auto fileStatus = CheckInstalledFileStatus(filePath, checkFileHash ? file.FileSha256 : Utility::SHA256::HashBuffer{}, fileHashes); installerStatus.Status.emplace_back( InstalledStatusType::DefaultInstallLocationFile, - filePath.string(), + filePath.u8string(), fileStatus); } } diff --git a/src/AppInstallerRepositoryCore/Public/winget/InstalledFilesCorrelation.h b/src/AppInstallerRepositoryCore/Public/winget/InstalledFilesCorrelation.h @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include <winget/Manifest.h> +#include <winget/FolderFileWatcher.h> +#include <optional> + +namespace AppInstaller::Repository::Correlation +{ + // TODO: This definition could be moved to Manifest when winget supports launch scenarios. + struct InstalledStartupLinkFile + { + // Relative file path to the startup menu folder. + // Same installers write the links to user startup folder or machine startup folder depending on the scope + // the installers were running for. So only relative file paths were collected. And seems enough. + AppInstaller::Manifest::string_t RelativeFilePath; + // Heuristic startup link type. + AppInstaller::Manifest::InstalledFileTypeEnum FileType = AppInstaller::Manifest::InstalledFileTypeEnum::Unknown; + }; + + struct InstallationMetadata + { + // Installed files metadata. Currently only capturing files pointed by a startup link. + AppInstaller::Manifest::InstallationMetadataInfo InstalledFiles; + // Startup links metadata. + std::vector<InstalledStartupLinkFile> StartupLinkFiles; + }; + + struct InstalledFilesCorrelation + { + // Constructor initializes the file watchers. + InstalledFilesCorrelation(); + virtual ~InstalledFilesCorrelation() = default; + + // Start the file watcher before the package installation. + virtual void StartFileWatcher(); + + // Stop the file watcher after the package installation. + virtual void StopFileWatcher(); + + // Correlates the given manifest against the data previously collected with capture calls. + virtual InstallationMetadata CorrelateForNewlyInstalled( + const Manifest::Manifest& manifest, + const std::string& arpInstallLocation); + + private: + struct FileWatcherFiles + { + // FileWatcher folder base. + std::filesystem::path Folder; + // List of files represented as relative file path to the base folder. + std::vector<std::filesystem::path> Files; + }; + + std::vector<AppInstaller::Utility::FolderFileWatcher> m_fileWatchers; + std::vector<FileWatcherFiles> m_files; + }; +} diff --git a/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h b/src/AppInstallerRepositoryCore/Public/winget/InstallerMetadataCollectionContext.h @@ -7,6 +7,7 @@ #include <winget/JsonUtil.h> #include <winget/ThreadGlobals.h> #include <winget/ARPCorrelation.h> +#include <winget/InstalledFilesCorrelation.h> #include <filesystem> #include <map> @@ -46,7 +47,14 @@ namespace AppInstaller::Repository::Metadata std::vector<Manifest::AppsAndFeaturesEntry> AppsAndFeaturesEntries; // 1.1 + // If Scope value is empty, the value is not set before. If the value is Unknown, a conflicting value is encountered. std::string Scope; + + // 1.2 + // If std::nullopt, the value is not set before. If the value is empty(i.e. !HasData()), a conflicting value is encountered. + std::optional<Manifest::InstallationMetadataInfo> InstalledFiles; + // If std::nullopt, the value is not set before. If the vector is empty, conflicting values are encountered. + std::optional<std::vector<Correlation::InstalledStartupLinkFile>> StartupLinkFiles; }; // Metadata from previous product revisions. @@ -82,7 +90,10 @@ namespace AppInstaller::Repository::Metadata struct InstallerMetadataCollectionContext { InstallerMetadataCollectionContext(); - InstallerMetadataCollectionContext(std::unique_ptr<Correlation::ARPCorrelationData> correlationData, const std::wstring& json); + InstallerMetadataCollectionContext( + std::unique_ptr<Correlation::ARPCorrelationData> correlationData, + std::unique_ptr<Correlation::InstalledFilesCorrelation> installedFilesCorrelation, + const std::wstring& json); InstallerMetadataCollectionContext(const InstallerMetadataCollectionContext&) = delete; InstallerMetadataCollectionContext& operator=(const InstallerMetadataCollectionContext&) = delete; @@ -147,6 +158,7 @@ namespace AppInstaller::Repository::Metadata Manifest::Manifest m_incomingManifest; std::unique_ptr<Correlation::ARPCorrelationData> m_correlationData; + std::unique_ptr<Correlation::InstalledFilesCorrelation> m_installedFilesCorrelation; // Output data enum class OutputStatus diff --git a/src/AppInstallerRepositoryCore/pch.h b/src/AppInstallerRepositoryCore/pch.h @@ -7,9 +7,12 @@ #include <urlmon.h> #include <appmodel.h> #include <winhttp.h> +#include <Shlwapi.h> +#include <Shlobj.h> #pragma warning( push ) -#pragma warning ( disable : 6001 6340 6388 ) +#pragma warning ( disable : 6001 6340 6387 6388 28196 ) +#include <wil/filesystem.h> #include <wil/resource.h> #include <wil/result.h> #include <wil/result_macros.h> @@ -39,6 +42,7 @@ #include <winrt/Windows.Foundation.Collections.h> #include <winrt/Windows.Management.Deployment.h> #include <winrt/Windows.Storage.h> +#include <wrl/client.h> #include <algorithm> #include <chrono> @@ -58,6 +62,7 @@ #include <thread> #include <tuple> #include <type_traits> +#include <unordered_set> #include <utility> #include <json/json.h> diff --git a/src/AppInstallerSharedLib/AppInstallerStrings.cpp b/src/AppInstallerSharedLib/AppInstallerStrings.cpp @@ -115,6 +115,16 @@ namespace AppInstaller::Utility return a.length() >= b.length() && CaseInsensitiveEquals(a.substr(0, b.length()), b); } + bool CaseInsensitiveContainsSubstring(std::string_view a, std::string_view b) + { + auto it = std::search( + a.begin(), a.end(), + b.begin(), b.end(), + [](char ch1, char ch2) { return std::tolower(ch1) == std::tolower(ch2); } + ); + return (it != a.end()); + } + bool ICUCaseInsensitiveEquals(std::string_view a, std::string_view b) { return FoldCase(a) == FoldCase(b); diff --git a/src/AppInstallerSharedLib/Public/AppInstallerStrings.h b/src/AppInstallerSharedLib/Public/AppInstallerStrings.h @@ -112,6 +112,10 @@ namespace AppInstaller::Utility // Use this if one of the values is a known value, and thus ToLower is sufficient. bool CaseInsensitiveStartsWith(std::string_view a, std::string_view b); + // Determines if string a contains string b. + // Use this if one of the values is a known value, and thus ToLower is sufficient. + bool CaseInsensitiveContainsSubstring(std::string_view a, std::string_view b); + // Compares the two UTF8 strings in a case insensitive manner, using ICU for case folding. bool ICUCaseInsensitiveEquals(std::string_view a, std::string_view b);