InstallerMetadataCollectionContext.cpp - winget-cli - Unnamed repository; edit this file 'description' to name the repository.

InstallerMetadataCollectionContext.cpp (67517B)
      1 // Copyright (c) Microsoft Corporation.
      2 // Licensed under the MIT License.
      3 #include "pch.h"
      4 #include "winget/InstallerMetadataCollectionContext.h"
      5 
      6 #include <AppInstallerDownloader.h>
      7 #include <AppInstallerErrors.h>
      8 #include <AppInstallerLogging.h>
      9 #include <AppInstallerFileLogger.h>
     10 #include <winget/TraceLogger.h>
     11 #include <AppInstallerStrings.h>
     12 #include <winget/JsonUtil.h>
     13 #include <winget/ManifestJSONParser.h>
     14 
     15 using namespace AppInstaller::Utility;
     16 
     17 namespace AppInstaller::Repository::Metadata
     18 {
     19     namespace
     20     {
     21         struct ProductMetadataFields_1_N
     22         {
     23             ProductMetadataFields_1_N(const Version& version)
     24             {
     25                 if (::AppInstaller::Utility::Version{ "1.1" } <= version)
     26                 {
     27                     SchemaVersion = L"1.1";
     28                     Scope = L"scope";
     29                 }
     30 
     31                 if (::AppInstaller::Utility::Version{ "1.2" } <= version)
     32                 {
     33                     SchemaVersion = L"1.2";
     34                     InstalledFiles = L"installedFiles";
     35                     DefaultInstallLocation = L"DefaultInstallLocation";
     36                     InstallationMetadataFiles = L"Files";
     37                     InstalledFileRelativeFilePath = L"RelativeFilePath";
     38                     InstalledFileSha256 = L"FileSha256";
     39                     InstalledFileType = L"FileType";
     40                     InstalledFileInvocationParameter = L"InvocationParameter";
     41                     InstalledFileDisplayName = L"DisplayName";
     42                     InstalledStartupLinks = L"startupLinks";
     43                     InstalledStartupLinkPath = L"RelativeFilePath";
     44                     InstalledStartupLinkType = L"FileType";
     45                     Icons = L"icons";
     46                     IconContent = L"IconContent";
     47                     IconSha256 = L"IconSha256";
     48                     IconFileType = L"IconFileType";
     49                     IconResolution = L"IconResolution";
     50                     IconTheme = L"IconTheme";
     51                 }
     52             }
     53 
     54             utility::string_t SchemaVersion = L"1.0";
     55 
     56             // 1.0
     57             utility::string_t ProductVersionMin = L"productVersionMin";
     58             utility::string_t ProductVersionMax = L"productVersionMax";
     59             utility::string_t Metadata = L"metadata";
     60             utility::string_t InstallerHash = L"installerHash";
     61             utility::string_t SubmissionIdentifier = L"submissionIdentifier";
     62             utility::string_t Version = L"version";
     63             utility::string_t AppsAndFeaturesEntries = L"AppsAndFeaturesEntries";
     64             utility::string_t Historical = L"historical";
     65 
     66             // AppsAndFeaturesEntry fields.
     67             utility::string_t DisplayName = L"DisplayName";
     68             utility::string_t Publisher = L"Publisher";
     69             utility::string_t DisplayVersion = L"DisplayVersion";
     70             utility::string_t ProductCode = L"ProductCode";
     71             utility::string_t UpgradeCode = L"UpgradeCode";
     72             utility::string_t InstallerType = L"InstallerType";
     73 
     74             utility::string_t VersionMin = L"versionMin";
     75             utility::string_t VersionMax = L"versionMax";
     76             utility::string_t Names = L"names";
     77             utility::string_t Publishers = L"publishers";
     78             utility::string_t ProductCodes = L"productCodes";
     79             utility::string_t UpgradeCodes = L"upgradeCodes";
     80 
     81             // 1.1
     82             utility::string_t Scope;
     83 
     84             // 1.2
     85 
     86             // Installed files
     87             utility::string_t InstalledFiles;
     88             utility::string_t DefaultInstallLocation;
     89             utility::string_t InstallationMetadataFiles;
     90             utility::string_t InstalledFileRelativeFilePath;
     91             utility::string_t InstalledFileSha256;
     92             utility::string_t InstalledFileType;
     93             utility::string_t InstalledFileInvocationParameter;
     94             utility::string_t InstalledFileDisplayName;
     95             // Startup links
     96             utility::string_t InstalledStartupLinks;
     97             utility::string_t InstalledStartupLinkPath;
     98             utility::string_t InstalledStartupLinkType;
     99             // Icons
    100             utility::string_t Icons;
    101             utility::string_t IconContent;
    102             utility::string_t IconSha256;
    103             utility::string_t IconFileType;
    104             utility::string_t IconResolution;
    105             utility::string_t IconTheme;
    106         };
    107 
    108         struct OutputFields_1_0
    109         {
    110             utility::string_t Version = L"version";
    111             utility::string_t SubmissionData = L"submissionData";
    112             utility::string_t InstallerHash = L"installerHash";
    113             utility::string_t Status = L"status";
    114             utility::string_t Metadata = L"metadata";
    115             utility::string_t Diagnostics = L"diagnostics";
    116         };
    117         
    118         struct DiagnosticFields
    119         {
    120             // Error case
    121             utility::string_t ErrorHR = L"errorHR";
    122             utility::string_t ErrorText = L"errorText";
    123 
    124             // Non-error case
    125             utility::string_t Reason = L"reason";
    126             utility::string_t ChangedEntryCount = L"changedEntryCount";
    127             utility::string_t MatchedEntryCount = L"matchedEntryCount";
    128             utility::string_t IntersectionCount = L"intersectionCount";
    129             utility::string_t CorrelationMeasures = L"correlationMeasures";
    130             utility::string_t Value = L"value";
    131             utility::string_t Name = L"name";
    132             utility::string_t Publisher = L"publisher";
    133         };
    134 
    135         std::string GetRequiredString(const web::json::value& value, const utility::string_t& field)
    136         {
    137             auto optString = AppInstaller::JSON::GetRawStringValueFromJsonNode(value, field);
    138             if (!optString)
    139             {
    140                 AICLI_LOG(Repo, Error, << "Required field '" << Utility::ConvertToUTF8(field) << "' was not present");
    141                 THROW_HR(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE);
    142             }
    143             return std::move(optString).value();
    144         }
    145 
    146         void AddFieldIfNotEmpty(web::json::value& value, const utility::string_t& field, std::string_view string)
    147         {
    148             if (!string.empty())
    149             {
    150                 value[field] = AppInstaller::JSON::GetStringValue(string);
    151             }
    152         }
    153 
    154         web::json::value CreateStringArray(const std::set<std::string>& values)
    155         {
    156             web::json::value result;
    157             size_t index = 0;
    158 
    159             for (const std::string& value : values)
    160             {
    161                 result[index++] = AppInstaller::JSON::GetStringValue(value);
    162             }
    163 
    164             return result;
    165         }
    166 
    167         bool AddIfNotPresentAndNotEmpty(std::set<std::string>& strings, const std::set<std::string>& filter, const std::string& string)
    168         {
    169             if (string.empty() || filter.find(string) != filter.end())
    170             {
    171                 return false;
    172             }
    173 
    174             strings.emplace(string);
    175             return true;
    176         }
    177 
    178         bool AddIfNotPresentAndNotEmpty(std::set<std::string>& strings, const std::string& string)
    179         {
    180             return AddIfNotPresentAndNotEmpty(strings, strings, string);
    181         }
    182 
    183         void AddIfNotPresent(std::set<std::string>& strings, std::set<std::string>& filter, const std::set<std::string>& inputs)
    184         {
    185             for (const std::string& input : inputs)
    186             {
    187                 if (AddIfNotPresentAndNotEmpty(strings, filter, input))
    188                 {
    189                     filter.emplace(input);
    190                 }
    191             }
    192         }
    193 
    194         void FilterAndAddToEntries(Manifest::AppsAndFeaturesEntry&& newEntry, std::vector<Manifest::AppsAndFeaturesEntry>& entries)
    195         {
    196             // Erase all duplicated data from the new entry
    197             for (const auto& entry : entries)
    198             {
    199 #define WINGET_ERASE_IF_SAME(_value_) if (entry._value_ == newEntry._value_) { newEntry._value_.clear(); }
    200                 WINGET_ERASE_IF_SAME(DisplayName);
    201                 WINGET_ERASE_IF_SAME(DisplayVersion);
    202                 WINGET_ERASE_IF_SAME(ProductCode);
    203                 WINGET_ERASE_IF_SAME(Publisher);
    204                 WINGET_ERASE_IF_SAME(UpgradeCode);
    205 #undef WINGET_ERASE_IF_SAME
    206 
    207                 if (entry.InstallerType == newEntry.InstallerType)
    208                 {
    209                     newEntry.InstallerType = Manifest::InstallerTypeEnum::Unknown;
    210                 }
    211             }
    212 
    213             // If anything remains, add it
    214             if (!newEntry.DisplayName.empty() || !newEntry.DisplayVersion.empty() || !newEntry.ProductCode.empty() ||
    215                 !newEntry.Publisher.empty() || !newEntry.UpgradeCode.empty() || newEntry.InstallerType != Manifest::InstallerTypeEnum::Unknown)
    216             {
    217                 entries.emplace_back(std::move(newEntry));
    218             }
    219         }
    220 
    221         std::optional<std::string> GetStringFromFutureSchema(const web::json::value& value, const utility::string_t& field)
    222         {
    223             if (field.empty())
    224             {
    225                 return {};
    226             }
    227 
    228             return AppInstaller::JSON::GetRawStringValueFromJsonNode(value, field);
    229         }
    230 
    231         void SetStringFromFutureSchema(web::json::value& json, const utility::string_t& field, std::string_view value)
    232         {
    233             if (!field.empty())
    234             {
    235                 json[field] = AppInstaller::JSON::GetStringValue(value);
    236             }
    237         }
    238 
    239         // For installed files merging, we remove conflicting entries, like scope. Indicating we are not certain some files will always be installed.
    240         void MergeInstalledFilesMetadata(Manifest::InstallationMetadataInfo& existing, const Manifest::InstallationMetadataInfo& incoming)
    241         {
    242             if (!Utility::CaseInsensitiveEquals(existing.DefaultInstallLocation, incoming.DefaultInstallLocation))
    243             {
    244                 existing.Clear();
    245                 return;
    246             }
    247 
    248             auto existingItr = existing.Files.begin();
    249             while (existingItr != existing.Files.end())
    250             {
    251                 auto itr = std::find_if(incoming.Files.begin(), incoming.Files.end(), [&](const Manifest::InstalledFile& entry)
    252                     {
    253                         return Utility::CaseInsensitiveEquals(existingItr->RelativeFilePath, entry.RelativeFilePath);
    254                     });
    255 
    256                 if (itr == incoming.Files.end())
    257                 {
    258                     existingItr = existing.Files.erase(existingItr);
    259                 }
    260                 else
    261                 {
    262                     if (existingItr->InvocationParameter != itr->InvocationParameter)
    263                     {
    264                         existingItr->InvocationParameter.clear();
    265                     }
    266                     if (!Utility::CaseInsensitiveEquals(existingItr->DisplayName, itr->DisplayName))
    267                     {
    268                         existingItr->DisplayName.clear();
    269                     }
    270                     if (!Utility::SHA256::AreEqual(existingItr->FileSha256, itr->FileSha256))
    271                     {
    272                         existingItr->FileSha256.clear();
    273                     }
    274                     if (existingItr->FileType != itr->FileType)
    275                     {
    276                         existingItr->FileType = Manifest::InstalledFileTypeEnum::Unknown;
    277                     }
    278 
    279                     ++existingItr;
    280                 }
    281             }
    282         }
    283 
    284         // For startup link files merging, we add non duplicate entries, like ProductCodes. Indicating possible startup links an installer could potentially add.
    285         void MergeStartupLinkFilesMetadata(std::vector<Correlation::InstalledStartupLinkFile>& existing, const std::vector<Correlation::InstalledStartupLinkFile>& incoming)
    286         {
    287             for (auto const& incomingEntry : incoming)
    288             {
    289                 auto itr = std::find_if(existing.begin(), existing.end(), [&](const Correlation::InstalledStartupLinkFile& entry)
    290                     {
    291                         return Utility::CaseInsensitiveEquals(incomingEntry.RelativeFilePath, entry.RelativeFilePath);
    292                     });
    293 
    294                 if (itr == existing.end())
    295                 {
    296                     existing.emplace_back(incomingEntry);
    297                 }
    298                 else if (itr->FileType != incomingEntry.FileType)
    299                 {
    300                     // Set conflicting file type to Unknown.
    301                     itr->FileType = AppInstaller::Manifest::InstalledFileTypeEnum::Unknown;
    302                 }
    303             }
    304         }
    305 
    306         // TODO: This method could be moved to rest response parser and reused when winget supports launch
    307         // scenarios (i.e. when startup links info are exposed in winget manifest).
    308         std::optional<std::vector<Correlation::InstalledStartupLinkFile>> DeserializeInstalledStartupLinks(
    309             const web::json::value& startupLinkFiles,
    310             const ProductMetadataFields_1_N& fields)
    311         {
    312             if (startupLinkFiles.is_null() || !startupLinkFiles.is_array())
    313             {
    314                 return {};
    315             }
    316 
    317             std::vector<Correlation::InstalledStartupLinkFile> startupLinks;
    318             for (auto const& startupLink : startupLinkFiles.as_array())
    319             {
    320                 Correlation::InstalledStartupLinkFile fileEntry;
    321 
    322                 std::optional<std::string> relativeFilePath = AppInstaller::JSON::GetRawStringValueFromJsonNode(startupLink, fields.InstalledStartupLinkPath);
    323                 if (!AppInstaller::JSON::IsValidNonEmptyStringValue(relativeFilePath))
    324                 {
    325                     AICLI_LOG(Repo, Error, << "Missing RelativeFilePath in Installed Startup Link Files.");
    326                     return {};
    327                 }
    328 
    329                 fileEntry.RelativeFilePath = std::move(*relativeFilePath);
    330 
    331                 std::optional<std::string> fileType = AppInstaller::JSON::GetRawStringValueFromJsonNode(startupLink, fields.InstalledStartupLinkType);
    332                 if (AppInstaller::JSON::IsValidNonEmptyStringValue(fileType))
    333                 {
    334                     fileEntry.FileType = Manifest::ConvertToInstalledFileTypeEnum(*fileType);
    335                 }
    336 
    337                 startupLinks.emplace_back(std::move(fileEntry));
    338             }
    339 
    340             return startupLinks;
    341         }
    342 
    343         std::vector<ExtractedIconInfo> DeserializeExtractedIcons(
    344             const web::json::value& icons,
    345             const ProductMetadataFields_1_N& fields)
    346         {
    347             if (icons.is_null() || !icons.is_array())
    348             {
    349                 return {};
    350             }
    351 
    352             std::vector<ExtractedIconInfo> result;
    353             for (auto const& iconInfo : icons.as_array())
    354             {
    355                 ExtractedIconInfo iconInfoEntry;
    356 
    357                 auto content = AppInstaller::JSON::GetRawStringValueFromJsonNode(iconInfo, fields.IconContent);
    358                 if (!AppInstaller::JSON::IsValidNonEmptyStringValue(content))
    359                 {
    360                     AICLI_LOG(Repo, Error, << "Missing IconContent in Extracted Icons.");
    361                     return {};
    362                 }
    363 
    364                 iconInfoEntry.IconContent = AppInstaller::JSON::Base64Decode(*content);
    365 
    366                 std::optional<std::string> sha256 = AppInstaller::JSON::GetRawStringValueFromJsonNode(iconInfo, fields.IconSha256);
    367                 if (AppInstaller::JSON::IsValidNonEmptyStringValue(sha256))
    368                 {
    369                     iconInfoEntry.IconSha256 = Utility::SHA256::ConvertToBytes(*sha256);
    370                 }
    371 
    372                 std::optional<std::string> fileType = AppInstaller::JSON::GetRawStringValueFromJsonNode(iconInfo, fields.IconFileType);
    373                 if (AppInstaller::JSON::IsValidNonEmptyStringValue(fileType))
    374                 {
    375                     iconInfoEntry.IconFileType = Manifest::ConvertToIconFileTypeEnum(*fileType);
    376                 }
    377 
    378                 std::optional<std::string> theme = AppInstaller::JSON::GetRawStringValueFromJsonNode(iconInfo, fields.IconTheme);
    379                 if (AppInstaller::JSON::IsValidNonEmptyStringValue(theme))
    380                 {
    381                     iconInfoEntry.IconTheme = Manifest::ConvertToIconThemeEnum(*theme);
    382                 }
    383 
    384                 std::optional<std::string> resolution = AppInstaller::JSON::GetRawStringValueFromJsonNode(iconInfo, fields.IconResolution);
    385                 if (AppInstaller::JSON::IsValidNonEmptyStringValue(resolution))
    386                 {
    387                     iconInfoEntry.IconResolution = Manifest::ConvertToIconResolutionEnum(*resolution);
    388                 }
    389 
    390                 result.emplace_back(std::move(iconInfoEntry));
    391             }
    392 
    393             return result;
    394         }
    395     }
    396 
    397     void ProductMetadata::Clear()
    398     {
    399         SchemaVersion = {};
    400         ProductVersionMin = {};
    401         ProductVersionMax = {};
    402         InstallerMetadataMap.clear();
    403         HistoricalMetadataList.clear();
    404     }
    405 
    406     void ProductMetadata::FromJson(const web::json::value& json)
    407     {
    408         Clear();
    409 
    410         utility::string_t versionFieldName = L"version";
    411 
    412         THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, json.is_null());
    413 
    414         SchemaVersion = Version{ GetRequiredString(json, versionFieldName) };
    415         AICLI_LOG(Repo, Info, << "Parsing metadata JSON version " << SchemaVersion.ToString());
    416 
    417         if (SchemaVersion.PartAt(0).Integer == 1)
    418         {
    419             FromJson_1_N(json);
    420         }
    421         else
    422         {
    423             AICLI_LOG(Repo, Error, << "Don't know how to handle metadata version " << SchemaVersion.ToString());
    424             THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE));
    425         }
    426 
    427         // Sort the historical data with oldest last (thus b < a)
    428         std::sort(HistoricalMetadataList.begin(), HistoricalMetadataList.end(),
    429             [](const HistoricalMetadata& a, const HistoricalMetadata& b) {
    430                 return b.ProductVersionMin < a.ProductVersionMin;
    431             });
    432     }
    433 
    434     web::json::value ProductMetadata::ToJson(const Utility::Version& schemaVersion, size_t maximumSizeInBytes)
    435     {
    436         SchemaVersion = schemaVersion;
    437         AICLI_LOG(Repo, Info, << "Creating metadata JSON version " << SchemaVersion.ToString());
    438 
    439         using ToJsonFunctionPointer = web::json::value(ProductMetadata::*)();
    440         ToJsonFunctionPointer toJsonFunction = nullptr;
    441 
    442         if (SchemaVersion.PartAt(0).Integer == 1)
    443         {
    444             toJsonFunction = &ProductMetadata::ToJson_1_N;
    445         }
    446         else
    447         {
    448             AICLI_LOG(Repo, Error, << "Don't know how to handle metadata version " << SchemaVersion.ToString());
    449             THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE));
    450         }
    451 
    452         // Constrain the result based on maximum size given
    453         web::json::value result = (this->*toJsonFunction)();
    454 
    455         while (maximumSizeInBytes)
    456         {
    457             // Determine current size
    458             std::ostringstream temp;
    459             result.serialize(temp);
    460 
    461             std::string tempStr = temp.str();
    462             if (tempStr.length() > maximumSizeInBytes)
    463             {
    464                 if (!DropOldestHistoricalData())
    465                 {
    466                     AICLI_LOG(Repo, Error, << "Could not remove any more historical data to get under " << maximumSizeInBytes << " bytes");
    467                     AICLI_LOG(Repo, Info, << "  Smallest size was " << tempStr.length() << " bytes with value:\n" << tempStr);
    468                     THROW_HR(HRESULT_FROM_WIN32(ERROR_FILE_TOO_LARGE));
    469                 }
    470                 result = (this->*toJsonFunction)();
    471             }
    472             else
    473             {
    474                 break;
    475             }
    476         }
    477 
    478         return result;
    479     }
    480 
    481     void ProductMetadata::CopyFrom(const ProductMetadata& source, std::string_view submissionIdentifier)
    482     {
    483         // If the source has no installer metadata, consider it empty
    484         if (source.InstallerMetadataMap.empty())
    485         {
    486             return;
    487         }
    488 
    489         // With the same submission, just copy over all of the data
    490         if (source.InstallerMetadataMap.begin()->second.SubmissionIdentifier == submissionIdentifier)
    491         {
    492             *this = source;
    493             return;
    494         }
    495 
    496         // This is a new submission, so we must move all of the data to historical and update the older historical data
    497         // First, create a new historical entry for the current metadata
    498         HistoricalMetadata currentHistory;
    499 
    500         currentHistory.ProductVersionMin = source.ProductVersionMin;
    501         currentHistory.ProductVersionMax = source.ProductVersionMax;
    502 
    503         for (const auto& metadataItem : source.InstallerMetadataMap)
    504         {
    505             for (const auto& entry : metadataItem.second.AppsAndFeaturesEntries)
    506             {
    507                 AddIfNotPresentAndNotEmpty(currentHistory.Names, entry.DisplayName);
    508                 AddIfNotPresentAndNotEmpty(currentHistory.Publishers, entry.Publisher);
    509                 AddIfNotPresentAndNotEmpty(currentHistory.ProductCodes, entry.ProductCode);
    510                 AddIfNotPresentAndNotEmpty(currentHistory.UpgradeCodes, entry.UpgradeCode);
    511             }
    512         }
    513 
    514         // Copy the data in so that we can continue using currentHistory to track all strings
    515         HistoricalMetadataList.emplace_back(currentHistory);
    516 
    517         // Now, copy over the other historical data, filtering out anything we have seen
    518         for (const auto& historical : source.HistoricalMetadataList)
    519         {
    520             HistoricalMetadata copied;
    521             copied.ProductVersionMin = historical.ProductVersionMin;
    522             copied.ProductVersionMax = historical.ProductVersionMax;
    523             AddIfNotPresent(copied.Names, currentHistory.Names, historical.Names);
    524             AddIfNotPresent(copied.Publishers, currentHistory.Publishers, historical.Publishers);
    525             AddIfNotPresent(copied.ProductCodes, currentHistory.ProductCodes, historical.ProductCodes);
    526             AddIfNotPresent(copied.UpgradeCodes, currentHistory.UpgradeCodes, historical.UpgradeCodes);
    527 
    528             if (!copied.Names.empty() || !copied.Publishers.empty() || !copied.ProductCodes.empty() || !copied.UpgradeCodes.empty())
    529             {
    530                 HistoricalMetadataList.emplace_back(std::move(copied));
    531             }
    532         }
    533     }
    534 
    535     void ProductMetadata::FromJson_1_N(const web::json::value& json)
    536     {
    537         AICLI_LOG(Repo, Info, << "Parsing metadata JSON " << SchemaVersion.ToString() << " fields");
    538 
    539         ProductMetadataFields_1_N fields{ SchemaVersion };
    540 
    541         auto productVersionMinString = AppInstaller::JSON::GetRawStringValueFromJsonNode(json, fields.ProductVersionMin);
    542         if (productVersionMinString)
    543         {
    544             ProductVersionMin = Version{ std::move(productVersionMinString).value() };
    545         }
    546 
    547         auto productVersionMaxString = AppInstaller::JSON::GetRawStringValueFromJsonNode(json, fields.ProductVersionMax);
    548         if (productVersionMaxString)
    549         {
    550             ProductVersionMax = Version{ std::move(productVersionMaxString).value() };
    551         }
    552 
    553         // The 1.0 version of metadata uses the 1.5 version of REST
    554         JSON::ManifestJSONParser parser{ Version{ "1.5" } };
    555 
    556         std::string submissionIdentifierVerification;
    557 
    558         auto metadataArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(json, fields.Metadata);
    559         if (metadataArray)
    560         {
    561             for (const auto& item : metadataArray->get())
    562             {
    563                 std::string installerHashString = GetRequiredString(item, fields.InstallerHash);
    564                 THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, InstallerMetadataMap.find(installerHashString) != InstallerMetadataMap.end());
    565 
    566                 InstallerMetadata installerMetadata;
    567 
    568                 installerMetadata.SubmissionIdentifier = GetRequiredString(item, fields.SubmissionIdentifier);
    569                 if (submissionIdentifierVerification.empty())
    570                 {
    571                     submissionIdentifierVerification = installerMetadata.SubmissionIdentifier;
    572                 }
    573                 else if (submissionIdentifierVerification != installerMetadata.SubmissionIdentifier)
    574                 {
    575                     AICLI_LOG(Repo, Error, << "Different submission identifiers found in metadata: '" <<
    576                         submissionIdentifierVerification << "' and '" << installerMetadata.SubmissionIdentifier << "'");
    577                     THROW_HR(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE);
    578                 }
    579 
    580                 auto appsAndFeatures = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(item, fields.AppsAndFeaturesEntries);
    581                 THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !appsAndFeatures);
    582                 installerMetadata.AppsAndFeaturesEntries = parser.DeserializeAppsAndFeaturesEntries(appsAndFeatures.value());
    583 
    584                 installerMetadata.Scope = GetStringFromFutureSchema(item, fields.Scope).value_or(std::string{});
    585 
    586                 if (!fields.InstalledFiles.empty())
    587                 {
    588                     auto installedFiles = AppInstaller::JSON::GetJsonValueFromNode(item, fields.InstalledFiles);
    589                     if (installedFiles)
    590                     {
    591                         installerMetadata.InstalledFiles = parser.DeserializeInstallationMetadata(installedFiles->get());
    592                     }
    593                 }
    594 
    595                 if (!fields.InstalledStartupLinks.empty())
    596                 {
    597                     auto startupLinks = AppInstaller::JSON::GetJsonValueFromNode(item, fields.InstalledStartupLinks);
    598                     if (startupLinks)
    599                     {
    600                         installerMetadata.StartupLinkFiles = DeserializeInstalledStartupLinks(startupLinks->get(), fields);
    601                     }
    602                 }
    603 
    604                 if (!fields.Icons.empty())
    605                 {
    606                     auto icons = AppInstaller::JSON::GetJsonValueFromNode(item, fields.Icons);
    607                     if (icons)
    608                     {
    609                         installerMetadata.Icons = DeserializeExtractedIcons(icons->get(), fields);
    610                     }
    611                 }
    612 
    613                 InstallerMetadataMap[installerHashString] = std::move(installerMetadata);
    614             }
    615         }
    616 
    617         auto historicalArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(json, fields.Historical);
    618         if (historicalArray)
    619         {
    620             for (const auto& item : historicalArray->get())
    621             {
    622                 HistoricalMetadata historicalMetadata;
    623 
    624                 historicalMetadata.ProductVersionMin = Version{ GetRequiredString(item, fields.VersionMin) };
    625                 historicalMetadata.ProductVersionMax = Version{ GetRequiredString(item, fields.VersionMax) };
    626                 historicalMetadata.Names = AppInstaller::JSON::GetRawStringSetFromJsonNode(item, fields.Names);
    627                 historicalMetadata.Publishers = AppInstaller::JSON::GetRawStringSetFromJsonNode(item, fields.Publishers);
    628                 historicalMetadata.ProductCodes = AppInstaller::JSON::GetRawStringSetFromJsonNode(item, fields.ProductCodes);
    629                 historicalMetadata.UpgradeCodes = AppInstaller::JSON::GetRawStringSetFromJsonNode(item, fields.UpgradeCodes);
    630 
    631                 HistoricalMetadataList.emplace_back(std::move(historicalMetadata));
    632             }
    633         }
    634     }
    635 
    636     web::json::value ProductMetadata::ToJson_1_N()
    637     {
    638         AICLI_LOG(Repo, Info, << "Creating metadata JSON " << SchemaVersion.ToString() << " fields");
    639 
    640         ProductMetadataFields_1_N fields{ SchemaVersion };
    641 
    642         web::json::value result;
    643 
    644         result[fields.Version] = web::json::value::string(fields.SchemaVersion);
    645         result[fields.ProductVersionMin] = AppInstaller::JSON::GetStringValue(ProductVersionMin.ToString());
    646         result[fields.ProductVersionMax] = AppInstaller::JSON::GetStringValue(ProductVersionMax.ToString());
    647 
    648         web::json::value metadataArray = web::json::value::array();
    649         size_t metadataItemIndex = 0;
    650         for (const auto& item : InstallerMetadataMap)
    651         {
    652             web::json::value itemValue;
    653 
    654             itemValue[fields.InstallerHash] = AppInstaller::JSON::GetStringValue(item.first);
    655             itemValue[fields.SubmissionIdentifier] = AppInstaller::JSON::GetStringValue(item.second.SubmissionIdentifier);
    656             SetStringFromFutureSchema(itemValue, fields.Scope, item.second.Scope);
    657             if (!fields.InstalledFiles.empty() && item.second.InstalledFiles.has_value())
    658             {
    659                 web::json::value installationMetadata;
    660 
    661                 installationMetadata[fields.DefaultInstallLocation] = AppInstaller::JSON::GetStringValue(item.second.InstalledFiles->DefaultInstallLocation);
    662 
    663                 web::json::value installedFilesArray = web::json::value::array();
    664                 size_t installedFileIndex = 0;
    665                 for (const auto& entry : item.second.InstalledFiles->Files)
    666                 {
    667                     web::json::value entryValue;
    668                     AddFieldIfNotEmpty(entryValue, fields.InstalledFileRelativeFilePath, entry.RelativeFilePath);
    669                     AddFieldIfNotEmpty(entryValue, fields.InstalledFileInvocationParameter, entry.InvocationParameter);
    670                     AddFieldIfNotEmpty(entryValue, fields.InstalledFileDisplayName, entry.DisplayName);
    671                     entryValue[fields.InstalledFileType] = AppInstaller::JSON::GetStringValue(Manifest::InstalledFileTypeToString(entry.FileType));
    672                     if (!entry.FileSha256.empty())
    673                     {
    674                         entryValue[fields.InstalledFileSha256] = AppInstaller::JSON::GetStringValue(SHA256::ConvertToString(entry.FileSha256));
    675                     }
    676                     installedFilesArray[installedFileIndex++] = std::move(entryValue);
    677                 }
    678                 installationMetadata[fields.InstallationMetadataFiles] = std::move(installedFilesArray);
    679 
    680                 itemValue[fields.InstalledFiles] = std::move(installationMetadata);
    681             }
    682 
    683             if (!fields.InstalledStartupLinks.empty() && item.second.StartupLinkFiles.has_value())
    684             {
    685                 web::json::value startupLinkFilesArray = web::json::value::array();
    686                 size_t startupLinkFileIndex = 0;
    687                 for (const auto& entry : item.second.StartupLinkFiles.value())
    688                 {
    689                     web::json::value entryValue;
    690                     entryValue[fields.InstalledStartupLinkPath] = AppInstaller::JSON::GetStringValue(entry.RelativeFilePath);
    691                     entryValue[fields.InstalledStartupLinkType] = AppInstaller::JSON::GetStringValue(Manifest::InstalledFileTypeToString(entry.FileType));
    692 
    693                     startupLinkFilesArray[startupLinkFileIndex++] = std::move(entryValue);
    694                 }
    695 
    696                 itemValue[fields.InstalledStartupLinks] = std::move(startupLinkFilesArray);
    697             }
    698 
    699             if (!fields.Icons.empty() && !item.second.Icons.empty())
    700             {
    701                 web::json::value iconsArray = web::json::value::array();
    702                 size_t iconIndex = 0;
    703                 for (const auto& entry : item.second.Icons)
    704                 {
    705                     web::json::value entryValue;
    706                     entryValue[fields.IconContent] = AppInstaller::JSON::GetStringValue(AppInstaller::JSON::Base64Encode(entry.IconContent));
    707                     if (!entry.IconSha256.empty())
    708                     {
    709                         entryValue[fields.IconSha256] = AppInstaller::JSON::GetStringValue(SHA256::ConvertToString(entry.IconSha256));
    710                     }
    711                     entryValue[fields.IconFileType] = AppInstaller::JSON::GetStringValue(Manifest::IconFileTypeToString(entry.IconFileType));
    712                     entryValue[fields.IconTheme] = AppInstaller::JSON::GetStringValue(Manifest::IconThemeToString(entry.IconTheme));
    713                     entryValue[fields.IconResolution] = AppInstaller::JSON::GetStringValue(Manifest::IconResolutionToString(entry.IconResolution));
    714 
    715                     iconsArray[iconIndex++] = std::move(entryValue);
    716                 }
    717 
    718                 itemValue[fields.Icons] = std::move(iconsArray);
    719             }
    720 
    721             web::json::value appsAndFeaturesArray = web::json::value::array();
    722             size_t appsAndFeaturesEntryIndex = 0;
    723             for (const auto& entry : item.second.AppsAndFeaturesEntries)
    724             {
    725                 web::json::value entryValue;
    726 
    727                 AddFieldIfNotEmpty(entryValue, fields.DisplayName, entry.DisplayName);
    728                 AddFieldIfNotEmpty(entryValue, fields.Publisher, entry.Publisher);
    729                 AddFieldIfNotEmpty(entryValue, fields.DisplayVersion, entry.DisplayVersion);
    730                 AddFieldIfNotEmpty(entryValue, fields.ProductCode, entry.ProductCode);
    731                 AddFieldIfNotEmpty(entryValue, fields.UpgradeCode, entry.UpgradeCode);
    732                 if (entry.InstallerType != Manifest::InstallerTypeEnum::Unknown)
    733                 {
    734                     entryValue[fields.InstallerType] = AppInstaller::JSON::GetStringValue(Manifest::InstallerTypeToString(entry.InstallerType));
    735                 }
    736 
    737                 appsAndFeaturesArray[appsAndFeaturesEntryIndex++] = std::move(entryValue);
    738             }
    739 
    740             itemValue[fields.AppsAndFeaturesEntries] = std::move(appsAndFeaturesArray);
    741 
    742             metadataArray[metadataItemIndex++] = std::move(itemValue);
    743         }
    744 
    745         result[fields.Metadata] = std::move(metadataArray);
    746 
    747         web::json::value historicalArray = web::json::value::array();
    748         size_t historicalItemIndex = 0;
    749         for (const auto& item : HistoricalMetadataList)
    750         {
    751             web::json::value itemValue;
    752 
    753             itemValue[fields.VersionMin] = AppInstaller::JSON::GetStringValue(item.ProductVersionMin.ToString());
    754             itemValue[fields.VersionMax] = AppInstaller::JSON::GetStringValue(item.ProductVersionMax.ToString());
    755             itemValue[fields.Names] = CreateStringArray(item.Names);
    756             itemValue[fields.Publishers] = CreateStringArray(item.Publishers);
    757             itemValue[fields.ProductCodes] = CreateStringArray(item.ProductCodes);
    758             itemValue[fields.UpgradeCodes] = CreateStringArray(item.UpgradeCodes);
    759 
    760             historicalArray[historicalItemIndex++] = std::move(itemValue);
    761         }
    762 
    763         result[fields.Historical] = std::move(historicalArray);
    764 
    765         return result;
    766     }
    767 
    768     bool ProductMetadata::DropOldestHistoricalData()
    769     {
    770         if (HistoricalMetadataList.empty())
    771         {
    772             return false;
    773         }
    774 
    775         HistoricalMetadataList.pop_back();
    776         return true;
    777     }
    778 
    779     InstallerMetadataCollectionContext::InstallerMetadataCollectionContext() :
    780         m_correlationData(std::make_unique<Correlation::ARPCorrelationData>()),
    781         m_installedFilesCorrelation(std::make_unique<Correlation::InstalledFilesCorrelation>())
    782     {}
    783 
    784     InstallerMetadataCollectionContext::InstallerMetadataCollectionContext(
    785         std::unique_ptr<Correlation::ARPCorrelationData> correlationData,
    786         std::unique_ptr<Correlation::InstalledFilesCorrelation> installedFilesCorrelation,
    787         const std::wstring& json) :
    788         m_correlationData(std::move(correlationData)), m_installedFilesCorrelation(std::move(installedFilesCorrelation))
    789     {
    790         auto threadGlobalsLifetime = InitializeLogging({});
    791         InitializePreinstallState(json);
    792     }
    793 
    794     std::unique_ptr<InstallerMetadataCollectionContext> InstallerMetadataCollectionContext::FromFile(const std::filesystem::path& file, const std::filesystem::path& logFile)
    795     {
    796         THROW_HR_IF(E_INVALIDARG, file.empty());
    797         THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND), !std::filesystem::exists(file));
    798 
    799         std::unique_ptr<InstallerMetadataCollectionContext> result = std::make_unique<InstallerMetadataCollectionContext>();
    800         auto threadGlobalsLifetime = result->InitializeLogging(logFile);
    801 
    802         AICLI_LOG(Repo, Info, << "Opening InstallerMetadataCollectionContext input file: " << file);
    803         std::ifstream fileStream{ file };
    804 
    805         auto content = ReadEntireStream(fileStream);
    806         // CppRestSdk's implementation of json parsing does not work with '\0', so trimming them here
    807         content.erase(std::find(content.begin(), content.end(), '\0'), content.end());
    808 
    809         result->InitializePreinstallState(ConvertToUTF16(content));
    810 
    811         return result;
    812     }
    813 
    814     std::unique_ptr<InstallerMetadataCollectionContext> InstallerMetadataCollectionContext::FromURI(std::wstring_view uri, const std::filesystem::path& logFile)
    815     {
    816         THROW_HR_IF(E_INVALIDARG, uri.empty());
    817 
    818         std::unique_ptr<InstallerMetadataCollectionContext> result = std::make_unique<InstallerMetadataCollectionContext>();
    819         auto threadGlobalsLifetime = result->InitializeLogging(logFile);
    820 
    821         std::string utf8Uri = ConvertToUTF8(uri);
    822         THROW_HR_IF(E_INVALIDARG, !IsUrlRemote(utf8Uri));
    823 
    824         AICLI_LOG(Repo, Info, << "Downloading InstallerMetadataCollectionContext input file: " << utf8Uri);
    825 
    826         std::ostringstream jsonStream;
    827         ProgressCallback emptyCallback;
    828 
    829         const int MaxRetryCount = 2;
    830         for (int retryCount = 0; retryCount < MaxRetryCount; ++retryCount)
    831         {
    832             try
    833             {
    834                 auto downloadHash = DownloadToStream(utf8Uri, jsonStream, DownloadType::InstallerMetadataCollectionInput, emptyCallback);
    835                 break;
    836             }
    837             catch (...)
    838             {
    839                 if (retryCount < MaxRetryCount - 1)
    840                 {
    841                     AICLI_LOG(Repo, Info, << "  Downloading InstallerMetadataCollectionContext input failed, waiting a bit and retrying...");
    842                     Sleep(500);
    843                 }
    844                 else
    845                 {
    846                     throw;
    847                 }
    848             }
    849         }
    850 
    851         result->InitializePreinstallState(ConvertToUTF16(jsonStream.str()));
    852 
    853         return result;
    854     }
    855 
    856     std::unique_ptr<InstallerMetadataCollectionContext> InstallerMetadataCollectionContext::FromJSON(const std::wstring& json, const std::filesystem::path& logFile)
    857     {
    858         THROW_HR_IF(E_INVALIDARG, json.empty());
    859 
    860         std::unique_ptr<InstallerMetadataCollectionContext> result = std::make_unique<InstallerMetadataCollectionContext>();
    861         auto threadGlobalsLifetime = result->InitializeLogging(logFile);
    862         result->InitializePreinstallState(json);
    863 
    864         return result;
    865     }
    866 
    867     void InstallerMetadataCollectionContext::Complete(const std::filesystem::path& output)
    868     {
    869         auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread();
    870 
    871         THROW_HR_IF(E_INVALIDARG, !output.has_filename());
    872 
    873         if (output.has_parent_path())
    874         {
    875             std::filesystem::create_directories(output.parent_path());
    876         }
    877 
    878         std::ofstream outputStream{ output };
    879         THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_OPEN_FAILED), !outputStream);
    880 
    881         CompleteWithThreadGlobalsSet(outputStream);
    882     }
    883 
    884     void InstallerMetadataCollectionContext::Complete(std::ostream& output)
    885     {
    886         auto threadGlobalsLifetime = m_threadGlobals.SetForCurrentThread();
    887         CompleteWithThreadGlobalsSet(output);
    888     }
    889 
    890     std::wstring InstallerMetadataCollectionContext::Merge(const std::wstring& json, size_t maximumSizeInBytes, const std::filesystem::path& logFile)
    891     {
    892         ThreadLocalStorage::WingetThreadGlobals threadGlobals;
    893         auto globalsLifetime = InitializeLogging(threadGlobals, logFile);
    894 
    895         AICLI_LOG(Repo, Info, << "Parsing input JSON:\n" << ConvertToUTF8(json));
    896 
    897         // Parse and validate JSON
    898         try
    899         {
    900             utility::string_t versionFieldName = L"version";
    901 
    902             web::json::value inputValue = web::json::value::parse(json);
    903 
    904             THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, inputValue.is_null());
    905 
    906             Version inputVersion = Version{ GetRequiredString(inputValue, versionFieldName) };
    907             AICLI_LOG(Repo, Info, << "Parsing input JSON version " << inputVersion.ToString());
    908 
    909             web::json::value mergedResult;
    910 
    911             if (inputVersion.PartAt(0).Integer == 1)
    912             {
    913                 mergedResult = Merge_1_0(inputValue, maximumSizeInBytes);
    914             }
    915             else
    916             {
    917                 AICLI_LOG(Repo, Error, << "Don't know how to handle version " << inputVersion.ToString());
    918                 THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE));
    919             }
    920 
    921             std::wostringstream outputStream;
    922             mergedResult.serialize(outputStream);
    923 
    924             return std::move(outputStream).str();
    925         }
    926         catch (const web::json::json_exception& exc)
    927         {
    928             AICLI_LOG(Repo, Error, << "Exception parsing input JSON: " << exc.what());
    929         }
    930 
    931         // We will return within the try or throw a non-json exception, so if we get here it was a json exception.
    932         THROW_HR(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE);
    933     }
    934 
    935     void InstallerMetadataCollectionContext::CompleteWithThreadGlobalsSet(std::ostream& output)
    936     {
    937         web::json::value outputJSON;
    938 
    939         if (!ContainsError())
    940         {
    941             try
    942             {
    943                 // Collect post-install system state
    944                 m_correlationData->CapturePostInstallSnapshot();
    945                 m_installedFilesCorrelation->StopFileWatcher();
    946 
    947                 ComputeOutputData();
    948 
    949                 // Construct output JSON
    950                 AICLI_LOG(Repo, Info, << "Creating output JSON version for input version " << m_inputVersion.ToString());
    951 
    952                 if (m_inputVersion.PartAt(0).Integer == 1)
    953                 {
    954                     // We only have one version currently, so use that as long as the major version is 1
    955                     outputJSON = CreateOutputJson_1_0();
    956                 }
    957                 else
    958                 {
    959                     AICLI_LOG(Repo, Error, << "Don't know how to output for version " << m_inputVersion.ToString());
    960                     THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE));
    961                 }
    962             }
    963             catch (...)
    964             {
    965                 CollectErrorDataFromException(std::current_exception());
    966             }
    967         }
    968 
    969         if (ContainsError())
    970         {
    971             // We only have one version currently
    972             outputJSON = CreateErrorJson_1_0();
    973         }
    974 
    975         // Write output
    976         outputJSON.serialize(output);
    977     }
    978 
    979     std::unique_ptr<ThreadLocalStorage::PreviousThreadGlobals> InstallerMetadataCollectionContext::InitializeLogging(ThreadLocalStorage::WingetThreadGlobals& threadGlobals, const std::filesystem::path& logFile)
    980     {
    981         auto threadGlobalsLifetime = threadGlobals.SetForCurrentThread();
    982 
    983         Logging::Log().SetLevel(Logging::Level::Info);
    984         Logging::Log().SetEnabledChannels(Logging::Channel::All);
    985         Logging::EnableWilFailureTelemetry();
    986         Logging::TraceLogger::Add();
    987 
    988         if (!logFile.empty())
    989         {
    990             Logging::FileLogger::Add(logFile);
    991         }
    992 
    993         Logging::Telemetry().SetCaller("installer-metadata-collection");
    994         Logging::Telemetry().LogStartup();
    995 
    996         return threadGlobalsLifetime;
    997     }
    998 
    999     std::unique_ptr<ThreadLocalStorage::PreviousThreadGlobals> InstallerMetadataCollectionContext::InitializeLogging(const std::filesystem::path& logFile)
   1000     {
   1001         return InitializeLogging(m_threadGlobals, logFile);
   1002     }
   1003 
   1004     void InstallerMetadataCollectionContext::InitializePreinstallState(const std::wstring& json)
   1005     {
   1006         try
   1007         {
   1008             AICLI_LOG(Repo, Info, << "Parsing input JSON:\n" << ConvertToUTF8(json));
   1009 
   1010             // Parse and validate JSON
   1011             try
   1012             {
   1013                 utility::string_t versionFieldName = L"version";
   1014 
   1015                 web::json::value inputValue = web::json::value::parse(json);
   1016 
   1017                 THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, inputValue.is_null());
   1018 
   1019                 m_inputVersion = Version{ GetRequiredString(inputValue, versionFieldName) };
   1020                 AICLI_LOG(Repo, Info, << "Parsing input JSON version " << m_inputVersion.ToString());
   1021 
   1022                 if (m_inputVersion.PartAt(0).Integer == 1)
   1023                 {
   1024                     // We only have one version currently, so use that as long as the major version is 1
   1025                     ParseInputJson_1_0(inputValue);
   1026                 }
   1027                 else
   1028                 {
   1029                     AICLI_LOG(Repo, Error, << "Don't know how to handle version " << m_inputVersion.ToString());
   1030                     THROW_HR(HRESULT_FROM_WIN32(ERROR_UNSUPPORTED_TYPE));
   1031                 }
   1032             }
   1033             catch (const web::json::json_exception& exc)
   1034             {
   1035                 AICLI_LOG(Repo, Error, << "Exception parsing input JSON: " << exc.what());
   1036                 throw;
   1037             }
   1038 
   1039             // Collect pre-install system state
   1040             m_correlationData->CapturePreInstallSnapshot();
   1041             m_installedFilesCorrelation->StartFileWatcher();
   1042         }
   1043         catch (...)
   1044         {
   1045             CollectErrorDataFromException(std::current_exception());
   1046         }
   1047     }
   1048 
   1049     void InstallerMetadataCollectionContext::ComputeOutputData()
   1050     {
   1051         // Copy the metadata from the current; this function takes care of moving data to historical if the submission is new.
   1052         m_outputMetadata.CopyFrom(m_currentMetadata, m_submissionIdentifier);
   1053 
   1054         Correlation::ARPCorrelationSettings settings;
   1055         std::string arpInstallLocation;
   1056         // As this code is typically run in a controlled environment, we can assume that a single value change is very likely the correct value.
   1057         settings.AllowSingleChange = true;
   1058 
   1059         // ARP entry correlation
   1060         Correlation::ARPCorrelationResult correlationResult = m_correlationData->CorrelateForNewlyInstalled(m_incomingManifest, settings);
   1061 
   1062         if (correlationResult.Package)
   1063         {
   1064             auto& package = correlationResult.Package;
   1065 
   1066             // Update min and max versions based on the version of the correlated package
   1067             Version packageVersion{ package->GetProperty(PackageVersionProperty::Version) };
   1068 
   1069             if (m_outputMetadata.ProductVersionMin.IsEmpty() || packageVersion < m_outputMetadata.ProductVersionMin)
   1070             {
   1071                 m_outputMetadata.ProductVersionMin = packageVersion;
   1072             }
   1073 
   1074             if (m_outputMetadata.ProductVersionMax.IsEmpty() || m_outputMetadata.ProductVersionMax < packageVersion)
   1075             {
   1076                 m_outputMetadata.ProductVersionMax = packageVersion;
   1077             }
   1078 
   1079             // Create the AppsAndFeaturesEntry that we need to add
   1080             Manifest::AppsAndFeaturesEntry newEntry;
   1081             auto packageMetadata = package->GetMetadata();
   1082 
   1083             // Arp installed location will be used in later installed files correlation.
   1084             arpInstallLocation = packageMetadata[PackageVersionMetadata::InstalledLocation];
   1085 
   1086             // TODO: Use some amount of normalization here to prevent things like versions being in the name from bloating the data
   1087             newEntry.DisplayName = package->GetProperty(PackageVersionProperty::Name).get();
   1088             newEntry.DisplayVersion = packageVersion.ToString();
   1089             if (packageMetadata.count(PackageVersionMetadata::InstalledType))
   1090             {
   1091                 newEntry.InstallerType = Manifest::ConvertToInstallerTypeEnum(packageMetadata[PackageVersionMetadata::InstalledType]);
   1092             }
   1093             auto productCodes = package->GetMultiProperty(PackageVersionMultiProperty::ProductCode);
   1094             if (!productCodes.empty())
   1095             {
   1096                 newEntry.ProductCode = std::move(productCodes[0]).get();
   1097             }
   1098             newEntry.Publisher = package->GetProperty(PackageVersionProperty::Publisher).get();
   1099             // TODO: Support upgrade code throughout the code base...
   1100 
   1101             Manifest::ScopeEnum scope = Manifest::ConvertToScopeEnum(packageMetadata[PackageVersionMetadata::InstalledScope]);
   1102 
   1103             // ARP entry icon extraction upon ARP correlation success
   1104             auto icons = ExtractIconFromArpEntry(newEntry.ProductCode, scope);
   1105 
   1106             // Add or update the metadata for the installer hash
   1107             auto itr = m_outputMetadata.InstallerMetadataMap.find(m_installerHash);
   1108 
   1109             if (itr == m_outputMetadata.InstallerMetadataMap.end())
   1110             {
   1111                 // New entry needed
   1112                 ProductMetadata::InstallerMetadata newMetadata;
   1113 
   1114                 newMetadata.SubmissionIdentifier = m_submissionIdentifier;
   1115                 newMetadata.AppsAndFeaturesEntries.emplace_back(std::move(newEntry));
   1116 
   1117                 if (scope != Manifest::ScopeEnum::Unknown)
   1118                 {
   1119                     newMetadata.Scope = Manifest::ScopeToString(scope);
   1120                 }
   1121 
   1122                 if (!icons.empty())
   1123                 {
   1124                     newMetadata.Icons = std::move(icons);
   1125                 }
   1126 
   1127                 m_outputMetadata.InstallerMetadataMap[m_installerHash] = std::move(newMetadata);
   1128             }
   1129             else
   1130             {
   1131                 if (itr->second.Scope.empty())
   1132                 {
   1133                     itr->second.Scope = Manifest::ScopeToString(scope);
   1134                 }
   1135                 // If there is a conflicting scope already present, force it to Unknown
   1136                 else if (scope != Manifest::ScopeEnum::Unknown && Manifest::ConvertToScopeEnum(itr->second.Scope) != scope)
   1137                 {
   1138                     itr->second.Scope = Manifest::ScopeToString(Manifest::ScopeEnum::Unknown);
   1139                 }
   1140 
   1141                 // We will always use the latest extracted icons upon confliction.
   1142                 if (!icons.empty())
   1143                 {
   1144                     itr->second.Icons = std::move(icons);
   1145                 }
   1146 
   1147                 // Existing entry for installer hash, add/update the entry
   1148                 FilterAndAddToEntries(std::move(newEntry), itr->second.AppsAndFeaturesEntries);
   1149             }
   1150         }
   1151 
   1152         // Installation files correlation
   1153         auto installationMetadata = m_installedFilesCorrelation->CorrelateForNewlyInstalled(m_incomingManifest, arpInstallLocation);
   1154 
   1155         if (installationMetadata.InstalledFiles.HasData() || !installationMetadata.StartupLinkFiles.empty())
   1156         {
   1157             // Add or update the metadata for the installer hash
   1158             auto itr = m_outputMetadata.InstallerMetadataMap.find(m_installerHash);
   1159 
   1160             if (itr == m_outputMetadata.InstallerMetadataMap.end())
   1161             {
   1162                 // New entry needed
   1163                 ProductMetadata::InstallerMetadata newMetadata;
   1164 
   1165                 newMetadata.SubmissionIdentifier = m_submissionIdentifier;
   1166 
   1167                 if (installationMetadata.InstalledFiles.HasData())
   1168                 {
   1169                     newMetadata.InstalledFiles = std::move(installationMetadata.InstalledFiles);
   1170                 }
   1171                 if (!installationMetadata.StartupLinkFiles.empty())
   1172                 {
   1173                     newMetadata.StartupLinkFiles = std::move(installationMetadata.StartupLinkFiles);
   1174                 }
   1175 
   1176                 m_outputMetadata.InstallerMetadataMap[m_installerHash] = std::move(newMetadata);
   1177             }
   1178             else
   1179             {
   1180                 // Add new or merge with existing entry
   1181                 if (installationMetadata.InstalledFiles.HasData())
   1182                 {
   1183                     if (!itr->second.InstalledFiles.has_value())
   1184                     {
   1185                         itr->second.InstalledFiles = std::move(installationMetadata.InstalledFiles);
   1186                     }
   1187                     else
   1188                     {
   1189                         MergeInstalledFilesMetadata(*(itr->second.InstalledFiles), installationMetadata.InstalledFiles);
   1190                     }
   1191                 }
   1192 
   1193                 if (!installationMetadata.StartupLinkFiles.empty())
   1194                 {
   1195                     if (!itr->second.StartupLinkFiles.has_value())
   1196                     {
   1197                         itr->second.StartupLinkFiles = std::move(installationMetadata.StartupLinkFiles);
   1198                     }
   1199                     else
   1200                     {
   1201                         MergeStartupLinkFilesMetadata(*(itr->second.StartupLinkFiles), installationMetadata.StartupLinkFiles);
   1202                     }
   1203                 }
   1204             }
   1205         }
   1206 
   1207         if (correlationResult.Package)
   1208         {
   1209             m_outputStatus = OutputStatus::Success;
   1210         }
   1211         else
   1212         {
   1213             m_outputStatus = OutputStatus::LowConfidence;
   1214         }
   1215 
   1216         // Create the diagnostics data, based on the other values from the correlation result.
   1217         DiagnosticFields fields;
   1218 
   1219         m_outputDiagnostics[fields.Reason] = AppInstaller::JSON::GetStringValue(correlationResult.Reason);
   1220         m_outputDiagnostics[fields.ChangedEntryCount] = web::json::value::number(static_cast<int64_t>(correlationResult.ChangesToARP));
   1221         m_outputDiagnostics[fields.MatchedEntryCount] = web::json::value::number(static_cast<int64_t>(correlationResult.MatchesInARP));
   1222         m_outputDiagnostics[fields.IntersectionCount] = web::json::value::number(static_cast<int64_t>(correlationResult.CountOfIntersectionOfChangesAndMatches));
   1223 
   1224         constexpr size_t MaximumDiagnosticMeasures = 10;
   1225         web::json::value measuresArray = web::json::value::array();
   1226         for (size_t i = 0; i < correlationResult.Measures.size() && i < MaximumDiagnosticMeasures; ++i)
   1227         {
   1228             web::json::value measureValue;
   1229             const auto& measure = correlationResult.Measures[i];
   1230 
   1231             measureValue[fields.Value] = web::json::value::number(measure.Measure);
   1232             measureValue[fields.Name] = AppInstaller::JSON::GetStringValue(measure.Package->GetProperty(PackageVersionProperty::Name));
   1233             measureValue[fields.Publisher] = AppInstaller::JSON::GetStringValue(measure.Package->GetProperty(PackageVersionProperty::Publisher));
   1234 
   1235             measuresArray[i] = std::move(measureValue);
   1236         }
   1237 
   1238         m_outputDiagnostics[fields.CorrelationMeasures] = std::move(measuresArray);
   1239     }
   1240 
   1241     void InstallerMetadataCollectionContext::ParseInputJson_1_0(web::json::value& input)
   1242     {
   1243         AICLI_LOG(Repo, Info, << "Parsing input JSON 1.0 fields");
   1244 
   1245         // Field names
   1246         utility::string_t metadataVersionFieldName = L"supportedMetadataVersion";
   1247         utility::string_t metadataFieldName = L"currentMetadata";
   1248         utility::string_t submissionDataFieldName = L"submissionData";
   1249         utility::string_t submissionIdentifierFieldName = L"submissionIdentifier";
   1250         utility::string_t packageDataFieldName = L"packageData";
   1251         utility::string_t installerHashFieldName = L"installerHash";
   1252         utility::string_t defaultLocaleFieldName = L"DefaultLocale";
   1253         utility::string_t localesFieldName = L"Locales";
   1254 
   1255         // root fields
   1256         m_supportedMetadataVersion = Version{ GetRequiredString(input, metadataVersionFieldName) };
   1257 
   1258         auto currentMetadataValue = AppInstaller::JSON::GetJsonValueFromNode(input, metadataFieldName);
   1259         if (currentMetadataValue)
   1260         {
   1261             m_currentMetadata.FromJson(currentMetadataValue.value());
   1262         }
   1263 
   1264         // submissionData fields
   1265         auto submissionDataValue = AppInstaller::JSON::GetJsonValueFromNode(input, submissionDataFieldName);
   1266         THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !submissionDataValue);
   1267         m_submissionData = submissionDataValue.value();
   1268 
   1269         m_submissionIdentifier = GetRequiredString(m_submissionData, submissionIdentifierFieldName);
   1270 
   1271         // packageData fields
   1272         auto packageDataValue = AppInstaller::JSON::GetJsonValueFromNode(input, packageDataFieldName);
   1273         THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !packageDataValue);
   1274 
   1275         m_installerHash = GetRequiredString(packageDataValue.value(), installerHashFieldName);
   1276 
   1277         // The 1.0 version of input uses the 1.5 version of REST
   1278         JSON::ManifestJSONParser parser{ Version{ "1.5" }};
   1279 
   1280         {
   1281             auto defaultLocaleValue = AppInstaller::JSON::GetJsonValueFromNode(packageDataValue.value(), defaultLocaleFieldName);
   1282             THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !defaultLocaleValue);
   1283 
   1284             auto defaultLocale = parser.DeserializeLocale(defaultLocaleValue.value());
   1285             THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE,
   1286                 !defaultLocale ||
   1287                 !defaultLocale->Contains(Manifest::Localization::PackageName) ||
   1288                 !defaultLocale->Contains(Manifest::Localization::Publisher));
   1289 
   1290             m_incomingManifest.DefaultLocalization = std::move(defaultLocale).value();
   1291 
   1292             auto localesArray = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(packageDataValue.value(), localesFieldName);
   1293             if (localesArray)
   1294             {
   1295                 for (const auto& locale : localesArray->get())
   1296                 {
   1297                     auto localization = parser.DeserializeLocale(locale);
   1298                     if (localization)
   1299                     {
   1300                         m_incomingManifest.Localizations.emplace_back(std::move(localization).value());
   1301                     }
   1302                 }
   1303             }
   1304         }
   1305     }
   1306 
   1307     web::json::value InstallerMetadataCollectionContext::CreateOutputJson_1_0()
   1308     {
   1309         AICLI_LOG(Repo, Info, << "Setting output JSON 1.0 fields");
   1310 
   1311         OutputFields_1_0 fields;
   1312 
   1313         web::json::value result;
   1314 
   1315         result[fields.Version] = web::json::value::string(L"1.0");
   1316         result[fields.SubmissionData] = m_submissionData;
   1317         result[fields.InstallerHash] = AppInstaller::JSON::GetStringValue(m_installerHash);
   1318 
   1319         // Limit output status to 1.0 known values
   1320         OutputStatus statusToUse = OutputStatus::Unknown;
   1321         if (m_outputStatus == OutputStatus::Success || m_outputStatus == OutputStatus::Error || m_outputStatus == OutputStatus::LowConfidence)
   1322         {
   1323             statusToUse = m_outputStatus;
   1324         }
   1325         result[fields.Status] = web::json::value::string(ToString(statusToUse));
   1326 
   1327         if (m_outputStatus == OutputStatus::Success)
   1328         {
   1329             result[fields.Metadata] = m_outputMetadata.ToJson(m_supportedMetadataVersion, 0);
   1330         }
   1331 
   1332         result[fields.Diagnostics] = m_outputDiagnostics;
   1333 
   1334         return result;
   1335     }
   1336 
   1337     utility::string_t InstallerMetadataCollectionContext::ToString(OutputStatus status)
   1338     {
   1339         switch (status)
   1340         {
   1341         case OutputStatus::Success: return L"Success";
   1342         case OutputStatus::Error: return L"Error";
   1343         case OutputStatus::LowConfidence: return L"LowConfidence";
   1344         }
   1345 
   1346         // For both the status value of Unknown and anything else
   1347         return L"Unknown";
   1348     }
   1349 
   1350     bool InstallerMetadataCollectionContext::ContainsError() const
   1351     {
   1352         return m_outputStatus == OutputStatus::Error;
   1353     }
   1354 
   1355     void InstallerMetadataCollectionContext::CollectErrorDataFromException(std::exception_ptr exception)
   1356     {
   1357         m_outputStatus = OutputStatus::Error;
   1358 
   1359         try
   1360         {
   1361             std::rethrow_exception(exception);
   1362         }
   1363         catch (const wil::ResultException& re)
   1364         {
   1365             m_errorHR = re.GetErrorCode();
   1366             m_errorText = GetUserPresentableMessage(re);
   1367         }
   1368         catch (const winrt::hresult_error& hre)
   1369         {
   1370             m_errorHR = hre.code();
   1371             m_errorText = GetUserPresentableMessage(hre);
   1372         }
   1373         catch (const std::exception& e)
   1374         {
   1375             m_errorHR = E_FAIL;
   1376             m_errorText = GetUserPresentableMessage(e);
   1377         }
   1378         catch (...)
   1379         {
   1380             m_errorHR = E_UNEXPECTED;
   1381             m_errorText = "An unexpected exception type was thrown.";
   1382         }
   1383     }
   1384 
   1385     web::json::value InstallerMetadataCollectionContext::CreateErrorJson_1_0()
   1386     {
   1387         AICLI_LOG(Repo, Info, << "Setting error JSON 1.0 fields");
   1388 
   1389         OutputFields_1_0 fields;
   1390         DiagnosticFields diagnosticFields;
   1391 
   1392         web::json::value result;
   1393 
   1394         result[fields.Version] = web::json::value::string(L"1.0");
   1395         result[fields.SubmissionData] = m_submissionData;
   1396         result[fields.InstallerHash] = AppInstaller::JSON::GetStringValue(m_installerHash);
   1397         result[fields.Status] = web::json::value::string(ToString(OutputStatus::Error));
   1398         result[fields.Metadata] = web::json::value::null();
   1399 
   1400         web::json::value error;
   1401 
   1402         error[diagnosticFields.ErrorHR] = web::json::value::number(static_cast<int64_t>(m_errorHR));
   1403         error[diagnosticFields.ErrorText] = AppInstaller::JSON::GetStringValue(m_errorText);
   1404 
   1405         result[fields.Diagnostics] = std::move(error);
   1406 
   1407         return result;
   1408     }
   1409 
   1410     web::json::value InstallerMetadataCollectionContext::Merge_1_0(web::json::value& input, size_t maximumSizeInBytes)
   1411     {
   1412         AICLI_LOG(Repo, Info, << "Merging 1.0 input metadatas");
   1413 
   1414         utility::string_t metadatasFieldName = L"metadatas";
   1415 
   1416         auto metadatasValue = AppInstaller::JSON::GetRawJsonArrayFromJsonNode(input, metadatasFieldName);
   1417         THROW_HR_IF(APPINSTALLER_CLI_ERROR_JSON_INVALID_FILE, !metadatasValue);
   1418 
   1419         std::vector<ProductMetadata> metadatas;
   1420         for (const auto& value : metadatasValue->get())
   1421         {
   1422             ProductMetadata current;
   1423             current.FromJson(value);
   1424             metadatas.emplace_back(std::move(current));
   1425         }
   1426 
   1427         THROW_HR_IF(E_NOT_SET, metadatas.empty());
   1428 
   1429         // Require that all merging values use the same submission
   1430         for (const ProductMetadata& metadata : metadatas)
   1431         {
   1432             const std::string& firstSubmission = metadatas[0].InstallerMetadataMap.begin()->second.SubmissionIdentifier;
   1433             const std::string& metadataSubmission = metadata.InstallerMetadataMap.begin()->second.SubmissionIdentifier;
   1434             if (firstSubmission != metadataSubmission)
   1435             {
   1436                 AICLI_LOG(Repo, Info, << "Found submission identifier mismatch: " << firstSubmission << " != " << metadataSubmission);
   1437                 THROW_HR(E_NOT_VALID_STATE);
   1438             }
   1439         }
   1440 
   1441         // Do the actual merging
   1442         ProductMetadata resultMetadata;
   1443 
   1444         // The historical data should be the same across the board, so we can just copy the first one.
   1445         resultMetadata.HistoricalMetadataList = metadatas[0].HistoricalMetadataList;
   1446 
   1447         for (const ProductMetadata& metadata : metadatas)
   1448         {
   1449             // Get the minimum and maximum versions from the individual values
   1450             if (resultMetadata.ProductVersionMin.IsEmpty() || metadata.ProductVersionMin < resultMetadata.ProductVersionMin)
   1451             {
   1452                 resultMetadata.ProductVersionMin = metadata.ProductVersionMin;
   1453             }
   1454 
   1455             if (resultMetadata.ProductVersionMax < metadata.ProductVersionMax)
   1456             {
   1457                 resultMetadata.ProductVersionMax = metadata.ProductVersionMax;
   1458             }
   1459 
   1460             if (resultMetadata.SchemaVersion < metadata.SchemaVersion)
   1461             {
   1462                 resultMetadata.SchemaVersion = metadata.SchemaVersion;
   1463             }
   1464 
   1465             for (const auto& installerMetadata : metadata.InstallerMetadataMap)
   1466             {
   1467                 auto itr = resultMetadata.InstallerMetadataMap.find(installerMetadata.first);
   1468                 if (itr == resultMetadata.InstallerMetadataMap.end())
   1469                 {
   1470                     // Installer hash not in the result, so just copy it
   1471                     resultMetadata.InstallerMetadataMap.emplace(installerMetadata);
   1472                 }
   1473                 else
   1474                 {
   1475                     if (itr->second.Scope.empty())
   1476                     {
   1477                         itr->second.Scope = installerMetadata.second.Scope;
   1478                     }
   1479                     else if (!installerMetadata.second.Scope.empty())
   1480                     {
   1481                         // If there is a conflicting scope already present, force it to Unknown
   1482                         if (Manifest::ConvertToScopeEnum(itr->second.Scope) != Manifest::ConvertToScopeEnum(installerMetadata.second.Scope))
   1483                         {
   1484                             itr->second.Scope = Manifest::ScopeToString(Manifest::ScopeEnum::Unknown);
   1485                         }
   1486                     }
   1487 
   1488                     // We will always use the latest extracted icons upon confliction.
   1489                     if (!installerMetadata.second.Icons.empty())
   1490                     {
   1491                         itr->second.Icons = installerMetadata.second.Icons;
   1492                     }
   1493 
   1494                     if (!itr->second.InstalledFiles.has_value())
   1495                     {
   1496                         itr->second.InstalledFiles = installerMetadata.second.InstalledFiles;
   1497                     }
   1498                     else if (installerMetadata.second.InstalledFiles.has_value())
   1499                     {
   1500                         MergeInstalledFilesMetadata(*(itr->second.InstalledFiles), *(installerMetadata.second.InstalledFiles));
   1501                     }
   1502 
   1503                     if (!itr->second.StartupLinkFiles.has_value())
   1504                     {
   1505                         itr->second.StartupLinkFiles = installerMetadata.second.StartupLinkFiles;
   1506                     }
   1507                     else if (installerMetadata.second.StartupLinkFiles.has_value())
   1508                     {
   1509                         MergeStartupLinkFilesMetadata(*(itr->second.StartupLinkFiles), *(installerMetadata.second.StartupLinkFiles));
   1510                     }
   1511 
   1512                     // Merge into existing installer data
   1513                     for (const auto& targetEntry : installerMetadata.second.AppsAndFeaturesEntries)
   1514                     {
   1515                         FilterAndAddToEntries(Manifest::AppsAndFeaturesEntry{ targetEntry }, itr->second.AppsAndFeaturesEntries);
   1516                     }
   1517                 }
   1518             }
   1519         }
   1520 
   1521         // Convert to JSON
   1522         return resultMetadata.ToJson(resultMetadata.SchemaVersion, maximumSizeInBytes);
   1523     }
   1524 }
	winget-cli Unnamed repository; edit this file 'description' to name the repository.
	Log \| Files \| Refs \| README \| LICENSE