winget-cli

Unnamed repository; edit this file 'description' to name the repository.
Log | Files | Refs | README | LICENSE

CompositeSource.cpp (82144B)


      1 // Copyright (c) Microsoft Corporation.
      2 // Licensed under the MIT License.
      3 #include "pch.h"
      4 #include "CompositeSource.h"
      5 #include <winget/ExperimentalFeature.h>
      6 
      7 using namespace AppInstaller::Settings;
      8 
      9 namespace AppInstaller::Repository
     10 {
     11     using namespace std::string_view_literals;
     12 
     13     namespace anon
     14     {
     15         Utility::VersionAndChannel GetVACFromVersion(IPackageVersion* packageVersion)
     16         {
     17             return {
     18                 Utility::Version(packageVersion->GetProperty(PackageVersionProperty::Version)),
     19                 Utility::Channel(packageVersion->GetProperty(PackageVersionProperty::Channel))
     20             };
     21         }
     22 
     23         // Returns true for fields that provide a strong match; one that is not based on a heuristic.
     24         bool IsStrongMatchField(PackageMatchField field)
     25         {
     26             switch (field)
     27             {
     28             case AppInstaller::Repository::PackageMatchField::PackageFamilyName:
     29             case AppInstaller::Repository::PackageMatchField::ProductCode:
     30             case AppInstaller::Repository::PackageMatchField::UpgradeCode:
     31                 return true;
     32             }
     33 
     34             return false;
     35         }
     36 
     37         // Gets the only available package from the composite, ensuring this fact in test contexts.
     38         std::shared_ptr<IPackage> OnlyAvailable(const std::shared_ptr<ICompositePackage>& composite)
     39         {
     40             std::vector<std::shared_ptr<IPackage>> availablePackages = composite->GetAvailable();
     41 
     42 #ifndef AICLI_DISABLE_TEST_HOOKS
     43             THROW_HR_IF(E_UNEXPECTED, availablePackages.size() != 1);
     44 #endif
     45 
     46             return std::move(availablePackages.front());
     47         }
     48 
     49         // Move returns if there is only one package in the matches that is strong; otherwise returns an empty value.
     50         std::shared_ptr<ICompositePackage> FindOnlyStrongMatchFieldResult(std::vector<ResultMatch>& matches)
     51         {
     52             std::shared_ptr<ICompositePackage> result;
     53 
     54             for (auto&& match : matches)
     55             {
     56                 AICLI_LOG(Repo, Info, << "  Checking match with package id: " << match.Package->GetProperty(PackageProperty::Id));
     57 
     58                 if (IsStrongMatchField(match.MatchCriteria.Field))
     59                 {
     60                     if (!result)
     61                     {
     62                         result = std::move(match.Package);
     63                     }
     64                     else
     65                     {
     66                         AICLI_LOG(Repo, Info, << "  Found multiple packages with strong match fields");
     67                         result.reset();
     68                         break;
     69                     }
     70                 }
     71             }
     72 
     73             return result;
     74         }
     75 
     76         // Gets a single matching package from the results
     77         template <typename MultipleIntro, typename Indeterminate>
     78         std::shared_ptr<ICompositePackage> GetMatchingPackage(std::vector<ResultMatch>& matches, MultipleIntro&& multipleIntro, Indeterminate&& indeterminate)
     79         {
     80             if (matches.empty())
     81             {
     82                 return {};
     83             }
     84             else if (matches.size() == 1)
     85             {
     86                 return std::move(matches[0].Package);
     87             }
     88             else
     89             {
     90                 multipleIntro();
     91 
     92                 auto result = FindOnlyStrongMatchFieldResult(matches);
     93 
     94                 if (!result)
     95                 {
     96                     indeterminate();
     97                 }
     98 
     99                 return result;
    100             }
    101         }
    102 
    103         // For a given package from a tracking catalog, get the latest write time.
    104         // Look at all versions rather than just the latest to account for the potential of downgrading.
    105         std::chrono::system_clock::time_point GetLatestTrackingWriteTime(
    106             const std::shared_ptr<IPackage>& trackingPackage)
    107         {
    108             std::chrono::system_clock::time_point result{};
    109 
    110             for (const auto& key : trackingPackage->GetVersionKeys())
    111             {
    112                 auto version = trackingPackage->GetVersion(key);
    113                 if (version)
    114                 {
    115                     auto metadata = version->GetMetadata();
    116                     auto itr = metadata.find(PackageVersionMetadata::TrackingWriteTime);
    117                     if (itr != metadata.end())
    118                     {
    119                         std::int64_t unixEpoch = 0;
    120                         try
    121                         {
    122                             unixEpoch = std::stoll(itr->second);
    123                         }
    124                         CATCH_LOG();
    125 
    126                         std::chrono::system_clock::time_point versionTime = Utility::ConvertUnixEpochToSystemClock(unixEpoch);
    127 
    128                         if (versionTime > result)
    129                         {
    130                             result = versionTime;
    131                         }
    132                     }
    133                 }
    134             }
    135 
    136             return result;
    137         }
    138 
    139         // An installed package's version reported in ARP does not necessarily match the versions used for the manifest.
    140         // This function uses the data in the manifest to map the installed version string to the version used by the manifest.
    141         //
    142         // TODO: Note: Currently this function assumes the all versions in the available package is from one source.
    143         // Even though a composite package can have available packages from multiple sources, we only call this function
    144         // for the default (first) available package. If we ever need to consider other sources, this function needs to be revisited.
    145         std::string GetMappedInstalledVersion(const std::string& installedVersion, const std::shared_ptr<IPackage>& availablePackage)
    146         {
    147             // Perform an initial check to see if the latest version has a mapping; if it does not, do not attempt any more.
    148             auto latestVersion = availablePackage->GetLatestVersion();
    149             if (latestVersion)
    150             {
    151                 auto version = latestVersion->GetProperty(PackageVersionProperty::Version);
    152                 auto arpMinVersion = latestVersion->GetProperty(PackageVersionProperty::ArpMinVersion);
    153                 auto arpMaxVersion = latestVersion->GetProperty(PackageVersionProperty::ArpMaxVersion);
    154 
    155                 if ((arpMinVersion.empty() || arpMinVersion == version) && (arpMaxVersion.empty() || arpMaxVersion == version))
    156                 {
    157                     return installedVersion;
    158                 }
    159             }
    160 
    161             // Stores raw versions value strings to run a preliminary check whether version mapping is needed.
    162             std::vector<std::tuple<std::string, std::string, std::string>> rawVersionValues;
    163             auto versionKeys = availablePackage->GetVersionKeys();
    164             bool shouldTryPerformMapping = false;
    165 
    166             for (auto const& versionKey : versionKeys)
    167             {
    168                 auto availableVersion = availablePackage->GetVersion(versionKey);
    169                 std::string arpMinVersion = availableVersion->GetProperty(PackageVersionProperty::ArpMinVersion);
    170                 std::string arpMaxVersion = availableVersion->GetProperty(PackageVersionProperty::ArpMaxVersion);
    171 
    172                 if (!arpMinVersion.empty() && !arpMaxVersion.empty())
    173                 {
    174                     std::string manifestVersion = versionKey.Version;
    175 
    176                     if (!shouldTryPerformMapping && (arpMinVersion != manifestVersion || arpMaxVersion != manifestVersion))
    177                     {
    178                         shouldTryPerformMapping = true;
    179                     }
    180 
    181                     rawVersionValues.emplace_back(std::make_tuple(std::move(manifestVersion), std::move(arpMinVersion), std::move(arpMaxVersion)));
    182                 }
    183             }
    184 
    185             if (!shouldTryPerformMapping)
    186             {
    187                 return installedVersion;
    188             }
    189 
    190             // Construct a map between manifest version and arp version range. The map is ordered in descending by package version.
    191             std::vector<std::pair<Utility::Version, Utility::VersionRange>> arpVersionMap;
    192 
    193             for (auto& tuple : rawVersionValues)
    194             {
    195                 auto&& [manifestVersion, arpMinVersion, arpMaxVersion] = std::move(tuple);
    196                 Utility::VersionRange arpVersionRange{ Utility::Version(std::move(arpMinVersion)), Utility::Version(std::move(arpMaxVersion)) };
    197                 Utility::Version manifestVer{ std::move(manifestVersion) };
    198                 // Skip mapping to unknown version
    199                 if (!manifestVer.IsUnknown())
    200                 {
    201                     arpVersionMap.emplace_back(std::make_pair(std::move(manifestVer), std::move(arpVersionRange)));
    202                 }
    203             }
    204 
    205             // Go through the arp version map and determine what mapping should be performed.
    206             // shouldPerformMapping is true when at least 1 arp version range is different from the package version.
    207             bool shouldPerformMapping = false;
    208             bool isArpVersionRangeInDescendingOrder = true;
    209             const Utility::VersionRange* previousVersionRange = nullptr;
    210 
    211             for (auto const& pair : arpVersionMap)
    212             {
    213                 // If arp version range is not same as package version, should perform mapping
    214                 // This check is still needed to account for 1.0 == 1.0.0 cases
    215                 if (!shouldPerformMapping && !pair.second.IsSameAsSingleVersion(pair.first))
    216                 {
    217                     shouldPerformMapping = true;
    218                 }
    219 
    220                 if (!previousVersionRange)
    221                 {
    222                     // This is the first non empty arp version range
    223                     previousVersionRange = &pair.second;
    224                 }
    225                 else if (isArpVersionRangeInDescendingOrder)
    226                 {
    227                     // The arp version range should be less than previous range
    228                     if (pair.second < *previousVersionRange)
    229                     {
    230                         previousVersionRange = &pair.second;
    231                     }
    232                     else
    233                     {
    234                         isArpVersionRangeInDescendingOrder = false;
    235                     }
    236                 }
    237             }
    238 
    239             // Now perform arp version mapping
    240             if (shouldPerformMapping)
    241             {
    242                 Utility::Version installed{ installedVersion };
    243                 for (auto const& pair : arpVersionMap)
    244                 {
    245                     // If the installed version is in the arp version range
    246                     if (pair.second.ContainsVersion(installed))
    247                     {
    248                         return pair.first.ToString();
    249                     }
    250                 }
    251 
    252                 // At this point, no mapping found. Perform approximate mapping if applicable.
    253                 // We'll start from end of the vector because we try to find closest less than version if possible.
    254                 if (isArpVersionRangeInDescendingOrder)
    255                 {
    256                     const Utility::Version* lastGreaterThanVersion = nullptr;
    257                     auto it = arpVersionMap.rbegin();
    258                     while (it != arpVersionMap.rend())
    259                     {
    260                         const auto& pair = *it;
    261                         if (installed < pair.second.GetMinVersion())
    262                         {
    263                             return Utility::Version{ pair.first, Utility::Version::ApproximateComparator::LessThan }.ToString();
    264                         }
    265                         else
    266                         {
    267                             lastGreaterThanVersion = &pair.first;
    268                         }
    269                         
    270                         it++;
    271                     }
    272 
    273                     // No approximate less than version found, approximate greater than version will be returned.
    274                     if (lastGreaterThanVersion)
    275                     {
    276                         return Utility::Version{ *lastGreaterThanVersion, Utility::Version::ApproximateComparator::GreaterThan }.ToString();
    277                     }
    278                 }
    279             }
    280 
    281             // return the input installed version if no mapping is performed or found.
    282             return installedVersion;
    283         }
    284 
    285         // A composite package installed version that allows us to override the source or the version.
    286         struct CompositeInstalledVersion : public IPackageVersion
    287         {
    288             CompositeInstalledVersion(std::shared_ptr<IPackageVersion> baseInstalledVersion, Source trackingSource, std::shared_ptr<IPackageVersion> trackingPackageVersion, std::string overrideVersion = {}) :
    289                 m_baseInstalledVersion(std::move(baseInstalledVersion)), m_trackingSource(std::move(trackingSource)), m_trackingPackageVersion(std::move(trackingPackageVersion)), m_overrideVersion(std::move(overrideVersion))
    290             {}
    291 
    292             Utility::LocIndString GetProperty(PackageVersionProperty property) const override
    293             {
    294                 // If there is an override version, use it.
    295                 if (property == PackageVersionProperty::Version && !m_overrideVersion.empty())
    296                 {
    297                     return Utility::LocIndString{ m_overrideVersion };
    298                 }
    299 
    300                 return m_baseInstalledVersion->GetProperty(property);
    301             }
    302 
    303             std::vector<Utility::LocIndString> GetMultiProperty(PackageVersionMultiProperty property) const override
    304             {
    305                 return m_baseInstalledVersion->GetMultiProperty(property);
    306             }
    307 
    308             Manifest::Manifest GetManifest() override
    309             {
    310                 return m_baseInstalledVersion->GetManifest();
    311             }
    312 
    313             Source GetSource() const override
    314             {
    315                 // If there is a tracking source, use it instead to indicate that it came from there.
    316                 if (m_trackingSource)
    317                 {
    318                     return m_trackingSource;
    319                 }
    320 
    321                 return m_baseInstalledVersion->GetSource();
    322             }
    323 
    324             Metadata GetMetadata() const override
    325             {
    326                 auto result = m_baseInstalledVersion->GetMetadata();
    327 
    328                 // Populate metadata from tracking package version if not present in base installed version.
    329                 if (m_trackingPackageVersion)
    330                 {
    331                     auto trackingMetadata = m_trackingPackageVersion->GetMetadata();
    332                     for (auto metadataItem : { PackageVersionMetadata::InstalledArchitecture, PackageVersionMetadata::InstalledLocale,
    333                         PackageVersionMetadata::UserIntentArchitecture, PackageVersionMetadata::UserIntentLocale, PackageVersionMetadata::PinnedState })
    334                     {
    335                         auto itr = trackingMetadata.find(metadataItem);
    336                         auto existingItr = result.find(metadataItem);
    337                         if (itr != trackingMetadata.end() && existingItr == result.end())
    338                         {
    339                             result[metadataItem] = itr->second;
    340                         }
    341                     }
    342                 }
    343 
    344                 return result;
    345             }
    346 
    347         private:
    348             std::shared_ptr<IPackageVersion> m_baseInstalledVersion;
    349             Source m_trackingSource;
    350             std::string m_overrideVersion;
    351             std::shared_ptr<IPackageVersion> m_trackingPackageVersion;
    352         };
    353 
    354         // An IPackage for the installed package of a CompositePackage.
    355         struct CompositeInstalledPackage : public IPackage
    356         {
    357             static constexpr IPackageType PackageType = IPackageType::CompositeInstalledPackage;
    358 
    359             CompositeInstalledPackage(std::shared_ptr<IPackage> package)
    360             {
    361                 AddPackageAndVersionKeyData(std::move(package));
    362             }
    363 
    364             Utility::LocIndString GetProperty(PackageProperty property) const override
    365             {
    366                 THROW_HR_IF(E_UNEXPECTED, m_packages.empty() || m_versionKeyData.empty());
    367 
    368                 // Use the highest version for package properties
    369                 return m_packages[m_versionKeyData[0].PackageIndex]->GetProperty(property);
    370             }
    371 
    372             std::vector<Utility::LocIndString> GetMultiProperty(PackageMultiProperty property) const override
    373             {
    374                 std::vector<Utility::LocIndString> result;
    375 
    376                 for (const auto& package : m_packages)
    377                 {
    378                     for (auto&& string : package->GetMultiProperty(property))
    379                     {
    380                         auto itr = std::lower_bound(result.begin(), result.end(), string);
    381 
    382                         if (itr == result.end() || *itr != string)
    383                         {
    384                             result.emplace(itr, std::move(string));
    385                         }
    386                     }
    387                 }
    388 
    389                 return result;
    390             }
    391 
    392             std::vector<PackageVersionKey> GetVersionKeys() const override
    393             {
    394                 return { m_versionKeyData.begin(), m_versionKeyData.end() };
    395             }
    396 
    397             std::shared_ptr<IPackageVersion> GetVersion(const PackageVersionKey& versionKey) const override
    398             {
    399                 std::shared_ptr<IPackageVersion> installedVersion;
    400                 std::string overrideVersion;
    401 
    402                 for (const VersionKeyData& key : m_versionKeyData)
    403                 {
    404                     if (key.IsMatch(versionKey))
    405                     {
    406                         installedVersion = key.InstalledVersion;
    407                         overrideVersion = key.Version;
    408                         break;
    409                     }
    410                 }
    411 
    412                 if (installedVersion)
    413                 {
    414                     // Get the appropriate tracking version or latest if it is not found.
    415                     // The tracking package uses the mapped version.
    416                     std::shared_ptr<IPackageVersion> trackingPackageVersion;
    417                     if (m_trackingPackage)
    418                     {
    419                         // Remove our use of the package id as source
    420                         PackageVersionKey versionKey_NoSource = versionKey;
    421                         versionKey_NoSource.SourceId.clear();
    422 
    423                         trackingPackageVersion = m_trackingPackage->GetVersion(versionKey_NoSource);
    424 
    425                         if (!trackingPackageVersion)
    426                         {
    427                             trackingPackageVersion = m_trackingPackage->GetLatestVersion();
    428                         }
    429                     }
    430 
    431                     return std::make_shared<CompositeInstalledVersion>(std::move(installedVersion), m_trackingSource, std::move(trackingPackageVersion), std::move(overrideVersion));
    432                 }
    433 
    434                 return nullptr;
    435             }
    436 
    437             std::shared_ptr<IPackageVersion> GetLatestVersion() const override
    438             {
    439                 return GetVersion({});
    440             }
    441 
    442             Source GetSource() const override
    443             {
    444                 // If there is a tracking source, use it instead to indicate that it came from there.
    445                 // Otherwise, all of the installed packages should be from the same source.
    446                 return m_trackingSource ? m_trackingSource : m_packages[0]->GetSource();
    447             }
    448 
    449             bool IsSame(const IPackage* other) const override
    450             {
    451                 const CompositeInstalledPackage* otherPackage = PackageCast<const CompositeInstalledPackage*>(other);
    452 
    453                 if (otherPackage)
    454                 {
    455                     if (m_packages.size() != otherPackage->m_packages.size())
    456                     {
    457                         return false;
    458                     }
    459 
    460                     for (const auto& subPackage : m_packages)
    461                     {
    462                         bool foundSame = false;
    463 
    464                         for (const auto& otherSubPackage : otherPackage->m_packages)
    465                         {
    466                             if (subPackage->IsSame(otherSubPackage.get()))
    467                             {
    468                                 foundSame = true;
    469                                 break;
    470                             }
    471                         }
    472 
    473                         if (!foundSame)
    474                         {
    475                             return false;
    476                         }
    477                     }
    478 
    479                     return true;
    480                 }
    481 
    482                 return false;
    483             }
    484 
    485             const void* CastTo(IPackageType type) const override
    486             {
    487                 if (type == PackageType)
    488                 {
    489                     return this;
    490                 }
    491 
    492                 return nullptr;
    493             }
    494 
    495             void SetTracking(
    496                 Source trackingSource,
    497                 std::shared_ptr<IPackage> trackingPackage,
    498                 std::chrono::system_clock::time_point trackingWriteTime)
    499             {
    500                 m_trackingSource = std::move(trackingSource);
    501                 m_trackingPackage = std::move(trackingPackage);
    502                 m_trackingWriteTime = trackingWriteTime;
    503             }
    504 
    505             Source GetTrackingSource() const
    506             {
    507                 return m_trackingSource;
    508             }
    509 
    510             const std::shared_ptr<IPackage>& GetTrackingPackage() const
    511             {
    512                 return m_trackingPackage;
    513             }
    514 
    515             std::chrono::system_clock::time_point GetTrackingPackageWriteTime() const
    516             {
    517                 return m_trackingWriteTime;
    518             }
    519 
    520             bool ContainsInstalledPackage(const IPackage* installedPackage) const
    521             {
    522                 for (const auto& package : m_packages)
    523                 {
    524                     if (package->IsSame(installedPackage))
    525                     {
    526                         return true;
    527                     }
    528                 }
    529 
    530                 return false;
    531             }
    532 
    533             void FoldInstalledIn(const std::shared_ptr<CompositeInstalledPackage>& other)
    534             {
    535                 for (const auto& package : other->m_packages)
    536                 {
    537                     AddPackageAndVersionKeyData(package);
    538                 }
    539             }
    540 
    541             // Set a version that will override the version string from the installed package
    542             void SetOverrideInstalledVersion(const std::shared_ptr<IPackage>& availablePackage)
    543             {
    544                 if (availablePackage)
    545                 {
    546                     m_availablePackageVersionOverride = availablePackage;
    547 
    548                     for (auto& key : m_versionKeyData)
    549                     {
    550                         if (Manifest::DoesInstallerTypeSupportArpVersionRange(key.InstalledType))
    551                         {
    552                             key.Version = GetMappedInstalledVersion(key.InstalledVersion->GetProperty(PackageVersionProperty::Version), availablePackage);
    553                         }
    554                     }
    555                 }
    556             }
    557 
    558             bool IsEmpty() const
    559             {
    560                 return m_versionKeyData.empty();
    561             }
    562 
    563         private:
    564             // Contains information about all of the version keys.
    565             // We use the `SourceId` field to store the installed package identifier so that we can disambiguate keys is they have the same version.
    566             struct VersionKeyData : public PackageVersionKey
    567             {
    568                 size_t PackageIndex;
    569                 std::shared_ptr<IPackageVersion> InstalledVersion;
    570                 Manifest::InstallerTypeEnum InstalledType;
    571                 Utility::VersionAndChannel VersionAndChannel;
    572 
    573                 bool operator<(const VersionKeyData& other) const
    574                 {
    575                     return VersionAndChannel < other.VersionAndChannel;
    576                 }
    577             };
    578 
    579             // Adds the package and version key data to the composite.
    580             // The version keys are then sorted so that the first (index 0) in the vector has the highest version.
    581             // Note that it may tied for highest version if, for instance, the same version is installed for different architectures.
    582             void AddPackageAndVersionKeyData(std::shared_ptr<IPackage> package)
    583             {
    584                 // We don't want this to happen, but it could. Rather than a crash, we will log it and move on.
    585                 if (!package)
    586                 {
    587                     AICLI_LOG(Repo, Verbose, << "AddPackageAndVersionKeyData called with an empty package");
    588                     return;
    589                 }
    590 
    591                 size_t packageIndex = m_packages.size();
    592                 std::string packageIdentifier = package->GetProperty(PackageProperty::Id);
    593                 bool versionAdded = false;
    594 
    595                 for (const auto& versionKey : package->GetVersionKeys())
    596                 {
    597                     VersionKeyData keyData{ versionKey };
    598 
    599                     keyData.PackageIndex = packageIndex;
    600                     keyData.InstalledVersion = package->GetVersion(versionKey);
    601 
    602                     if (!keyData.InstalledVersion)
    603                     {
    604                         AICLI_LOG(Repo, Verbose, << "AddPackageAndVersionKeyData: Package [" << packageIdentifier << "] did not return a version for [" << versionKey.Version << "]");
    605                         continue;
    606                     }
    607 
    608                     // We use the `SourceId` field to store the installed package identifier so that we can disambiguate keys if they have the same version.
    609                     keyData.SourceId = packageIdentifier;
    610 
    611                     keyData.InstalledType = Manifest::ConvertToInstallerTypeEnum(keyData.InstalledVersion->GetMetadata()[PackageVersionMetadata::InstalledType]);
    612                     if (m_availablePackageVersionOverride && Manifest::DoesInstallerTypeSupportArpVersionRange(keyData.InstalledType))
    613                     {
    614                         keyData.Version = GetMappedInstalledVersion(keyData.InstalledVersion->GetProperty(PackageVersionProperty::Version), m_availablePackageVersionOverride);
    615                     }
    616 
    617                     keyData.VersionAndChannel = Utility::VersionAndChannel{ keyData.Version, keyData.Channel };
    618 
    619                     m_versionKeyData.emplace_back(std::move(keyData));
    620                     versionAdded = true;
    621                 }
    622 
    623                 if (versionAdded)
    624                 {
    625                     m_packages.emplace_back(std::move(package));
    626 
    627                     std::sort(m_versionKeyData.begin(), m_versionKeyData.end());
    628                 }
    629             }
    630 
    631             std::vector<std::shared_ptr<IPackage>> m_packages;
    632             std::vector<VersionKeyData> m_versionKeyData;
    633             Source m_trackingSource;
    634             std::shared_ptr<IPackage> m_trackingPackage;
    635             std::chrono::system_clock::time_point m_trackingWriteTime = std::chrono::system_clock::time_point::min();
    636             std::shared_ptr<IPackage> m_availablePackageVersionOverride;
    637         };
    638 
    639         // An ICompositePackage for the CompositeSource.
    640         struct CompositePackage : public ICompositePackage
    641         {
    642             // The availablePackage may only contain one available package within it, as it is expected to be the output of a search on a single source.
    643             CompositePackage(const std::shared_ptr<ICompositePackage>& installedPackage, const std::shared_ptr<ICompositePackage>& availablePackage = {}, bool setPrimary = false)
    644             {
    645                 if (installedPackage)
    646                 {
    647                     m_installedPackage = std::make_shared<CompositeInstalledPackage>(installedPackage->GetInstalled());
    648 
    649                     // If the installed package result existed, but didn't actually create any installed versions, drop it.
    650                     if (m_installedPackage->IsEmpty())
    651                     {
    652                         m_installedPackage.reset();
    653                     }
    654                 }
    655 
    656                 AddAvailablePackage(availablePackage, setPrimary);
    657             }
    658 
    659             Utility::LocIndString GetProperty(PackageProperty property) const override
    660             {
    661                 IPackage* truth = nullptr;
    662                 if (m_primaryAvailablePackage)
    663                 {
    664                     truth = m_primaryAvailablePackage.get();
    665                 }
    666                 if (!truth && !m_availablePackages.empty())
    667                 {
    668                     truth = m_availablePackages[0].get();
    669                 }
    670                 if (!truth)
    671                 {
    672                     truth = m_installedPackage.get();
    673                 }
    674 
    675                 THROW_HR_IF(E_UNEXPECTED, !truth);
    676 
    677                 return truth->GetProperty(property);
    678             }
    679 
    680             std::shared_ptr<IPackage> GetInstalled() override
    681             {
    682                 return m_installedPackage;
    683             }
    684 
    685             std::vector<std::shared_ptr<IPackage>> GetAvailable() override
    686             {
    687                 return m_availablePackages;
    688             }
    689 
    690             const std::vector<std::shared_ptr<IPackage>>& GetAvailablePackages()
    691             {
    692                 return m_availablePackages;
    693             }
    694 
    695             bool IsSameAsAnyAvailable(const IPackage* other) const
    696             {
    697                 if (other)
    698                 {
    699                     for (const auto& availablePackage : m_availablePackages)
    700                     {
    701                         if (other->IsSame(availablePackage.get()))
    702                         {
    703                             return true;
    704                         }
    705                     }
    706                 }
    707 
    708                 return false;
    709             }
    710 
    711             const std::shared_ptr<CompositeInstalledPackage>& GetInstalledPackage() const
    712             {
    713                 return m_installedPackage;
    714             }
    715 
    716             bool ContainsInstalledPackage(const IPackage* installedPackage) const
    717             {
    718                 return m_installedPackage ? m_installedPackage->ContainsInstalledPackage(installedPackage) : false;
    719             }
    720 
    721             void AddAvailablePackage(const std::shared_ptr<ICompositePackage>& availablePackage, bool setPrimary = false)
    722             {
    723                 if (availablePackage)
    724                 {
    725                     m_availablePackages.emplace_back(OnlyAvailable(availablePackage));
    726 
    727                     if (setPrimary)
    728                     {
    729                         m_primaryAvailablePackage = m_availablePackages.back();
    730                     }
    731 
    732                     // Set override for primary or with the first available version found
    733                     if (setPrimary || m_availablePackages.size() == 1)
    734                     {
    735                         TrySetOverrideInstalledVersion(m_availablePackages.back());
    736                     }
    737                 }
    738             }
    739 
    740             std::shared_ptr<IPackage>& GetPrimaryAvailablePackage()
    741             {
    742                 return m_primaryAvailablePackage;
    743             }
    744 
    745             Source GetTrackingSource() const
    746             {
    747                 return m_installedPackage ? m_installedPackage->GetTrackingSource() : Source{};
    748             }
    749 
    750             std::shared_ptr<IPackage> GetTrackingPackage() const
    751             {
    752                 return m_installedPackage ? m_installedPackage->GetTrackingPackage() : std::shared_ptr<IPackage>{};
    753             }
    754 
    755             std::chrono::system_clock::time_point GetTrackingPackageWriteTime() const
    756             {
    757                 return m_installedPackage ? m_installedPackage->GetTrackingPackageWriteTime() : std::chrono::system_clock::time_point::min();
    758             }
    759 
    760             void SetTracking(
    761                 Source trackingSource,
    762                 std::shared_ptr<IPackage> trackingPackage,
    763                 std::chrono::system_clock::time_point trackingWriteTime)
    764             {
    765                 if (m_installedPackage)
    766                 {
    767                     m_installedPackage->SetTracking(std::move(trackingSource), std::move(trackingPackage), trackingWriteTime);
    768                 }
    769             }
    770 
    771             void FoldInstalledIn(const std::shared_ptr<CompositePackage>& other)
    772             {
    773                 if (other->m_installedPackage)
    774                 {
    775                     if (m_installedPackage)
    776                     {
    777                         m_installedPackage->FoldInstalledIn(other->m_installedPackage);
    778                     }
    779                     else
    780                     {
    781                         m_installedPackage = other->m_installedPackage;
    782                     }
    783                 }
    784             }
    785 
    786         private:
    787             // Try to set a version that will override the version string from the installed package
    788             void TrySetOverrideInstalledVersion(const std::shared_ptr<IPackage>& availablePackage)
    789             {
    790                 if (m_installedPackage && availablePackage)
    791                 {
    792                     m_installedPackage->SetOverrideInstalledVersion(availablePackage);
    793                 }
    794             }
    795 
    796             std::shared_ptr<CompositeInstalledPackage> m_installedPackage;
    797             std::shared_ptr<IPackage> m_primaryAvailablePackage;
    798             std::vector<std::shared_ptr<IPackage>> m_availablePackages;
    799         };
    800 
    801         // The comparator compares the ResultMatch by MatchType first, then Field in a predefined order.
    802         struct ResultMatchComparator
    803         {
    804             template <typename U, typename V>
    805             bool operator() (
    806                 const U& match1,
    807                 const V& match2)
    808             {
    809                 if (match1.MatchCriteria.Type != match2.MatchCriteria.Type)
    810                 {
    811                     return match1.MatchCriteria.Type < match2.MatchCriteria.Type;
    812                 }
    813 
    814                 if (match1.MatchCriteria.Field != match2.MatchCriteria.Field)
    815                 {
    816                     return match1.MatchCriteria.Field < match2.MatchCriteria.Field;
    817                 }
    818 
    819                 return false;
    820             }
    821         };
    822 
    823         template <typename T>
    824         void SortResultMatches(std::vector<T>& matches)
    825         {
    826             std::stable_sort(matches.begin(), matches.end(), ResultMatchComparator());
    827         }
    828 
    829         // A copy of the standard match that holds a CompositePackage instead.
    830         struct CompositeResultMatch
    831         {
    832             std::shared_ptr<CompositePackage> Package;
    833             PackageMatchFilter MatchCriteria;
    834 
    835             CompositeResultMatch(std::shared_ptr<CompositePackage> p, PackageMatchFilter f) : Package(std::move(p)), MatchCriteria(std::move(f)) {}
    836         };
    837 
    838         // Stores data to enable correlation between installed and available packages.
    839         struct CompositeResult
    840         {
    841             // A system reference string.
    842             struct SystemReferenceString
    843             {
    844                 SystemReferenceString(PackageMatchField field, Utility::LocIndString string) :
    845                     Field(field), String1(Utility::FoldCase(string)) {}
    846 
    847                 SystemReferenceString(PackageMatchField field, Utility::LocIndString string1, Utility::LocIndString string2) :
    848                     Field(field), String1(Utility::FoldCase(string1)), String2(Utility::FoldCase(string2)) {}
    849 
    850                 bool operator<(const SystemReferenceString& other) const
    851                 {
    852                     if (Field != other.Field)
    853                     {
    854                         return Field < other.Field;
    855                     }
    856 
    857                     if (String1 != other.String1)
    858                     {
    859                         return String1 < other.String1;
    860                     }
    861 
    862                     return String2 < other.String2;
    863                 }
    864 
    865                 bool operator==(const SystemReferenceString& other) const
    866                 {
    867                     return Field == other.Field && String1 == other.String1 && String2 == other.String2;
    868                 }
    869 
    870                 void AddToFilters(
    871                     std::vector<PackageMatchFilter>& filters) const
    872                 {
    873                     switch (Field)
    874                     {
    875                     case PackageMatchField::NormalizedNameAndPublisher:
    876                         filters.emplace_back(PackageMatchFilter(Field, MatchType::Exact, String1.get(), String2.get()));
    877                         break;
    878 
    879                     default:
    880                         filters.emplace_back(PackageMatchFilter(Field, MatchType::Exact, String1.get()));
    881                     }
    882                 }
    883 
    884             private:
    885                 PackageMatchField Field;
    886                 Utility::LocIndString String1;
    887                 Utility::LocIndString String2;
    888             };
    889 
    890             // Data relevant to correlation for a package.
    891             struct PackageData
    892             {
    893                 std::set<SystemReferenceString> SystemReferenceStrings;
    894 
    895                 void AddIfNotPresent(SystemReferenceString&& srs)
    896                 {
    897                     if (SystemReferenceStrings.find(srs) == SystemReferenceStrings.end())
    898                     {
    899                         SystemReferenceStrings.emplace(std::move(srs));
    900                     }
    901                 }
    902 
    903                 SearchRequest CreateInclusionsSearchRequest(SearchPurpose searchPurpose) const
    904                 {
    905                     SearchRequest result;
    906                     for (const auto& srs : SystemReferenceStrings)
    907                     {
    908                         srs.AddToFilters(result.Inclusions);
    909                     }
    910                     result.Purpose = searchPurpose;
    911                     return result;
    912                 }
    913 
    914                 std::shared_ptr<IPackage> AddSystemReferenceStringsFromTrackingPackage(const PackageTrackingCatalog& trackingCatalog, const Utility::LocIndString& identifier, std::string_view sourceIdentifier)
    915                 {
    916                     SearchRequest trackingRequest;
    917                     trackingRequest.Filters.emplace_back(PackageMatchField::Id, MatchType::CaseInsensitive, identifier.get());
    918 
    919                     SearchResult trackingResult = trackingCatalog.Search(trackingRequest);
    920                     std::shared_ptr<IPackage> result;
    921 
    922                     if (trackingResult.Matches.size() == 1)
    923                     {
    924                         result = OnlyAvailable(trackingResult.Matches[0].Package);
    925                         AddSystemReferenceStrings(result.get());
    926                     }
    927                     else if (trackingResult.Matches.size() > 1)
    928                     {
    929                         AICLI_LOG(Repo, Warning, << "Found " << trackingResult.Matches.size() << " results for Id [" << identifier << "] in tracking catalog for: " << sourceIdentifier);
    930                     }
    931 
    932                     return result;
    933                 }
    934 
    935                 void AddSystemReferenceStrings(IPackage* package)
    936                 {
    937                     GetSystemReferenceStrings(
    938                         package,
    939                         PackageMultiProperty::PackageFamilyName,
    940                         PackageMatchField::PackageFamilyName);
    941 
    942                     GetSystemReferenceStrings(
    943                         package,
    944                         PackageMultiProperty::ProductCode,
    945                         PackageMatchField::ProductCode);
    946 
    947                     GetSystemReferenceStrings(
    948                         package,
    949                         PackageMultiProperty::UpgradeCode,
    950                         PackageMatchField::UpgradeCode);
    951 
    952                     GetNameAndPublisher(
    953                         package);
    954                 }
    955 
    956                 void AddSystemReferenceStringsFromManifest(const Manifest::Manifest& manifest)
    957                 {
    958                     for (const auto& pfn : manifest.GetPackageFamilyNames())
    959                     {
    960                         AddIfNotPresent(SystemReferenceString{ PackageMatchField::PackageFamilyName, Utility::LocIndString{ pfn } });
    961                     }
    962                     for (const auto& productCode : manifest.GetProductCodes())
    963                     {
    964                         AddIfNotPresent(SystemReferenceString{ PackageMatchField::ProductCode, Utility::LocIndString{ productCode } });
    965                     }
    966                     for (const auto& upgradeCode : manifest.GetUpgradeCodes())
    967                     {
    968                         AddIfNotPresent(SystemReferenceString{ PackageMatchField::UpgradeCode, Utility::LocIndString{ upgradeCode } });
    969                     }
    970                     for (const auto& name : manifest.GetPackageNames())
    971                     {
    972                         for (const auto& publisher : manifest.GetPublishers())
    973                         {
    974                             AddIfNotPresent(SystemReferenceString{
    975                                 PackageMatchField::NormalizedNameAndPublisher,
    976                                 Utility::LocIndString{ name },
    977                                 Utility::LocIndString{ publisher } });
    978                         }
    979                     }
    980                 }
    981 
    982             private:
    983                 void GetSystemReferenceStrings(
    984                     IPackage* package,
    985                     PackageMultiProperty prop,
    986                     PackageMatchField field)
    987                 {
    988                     for (auto&& string : package->GetMultiProperty(prop))
    989                     {
    990                         AddIfNotPresent(SystemReferenceString{ field, std::move(string) });
    991                     }
    992                 }
    993 
    994                 void GetNameAndPublisher(
    995                     IPackage* package)
    996                 {
    997                     // Unfortunately the names and publishers are unique and not tied to each other strictly, so we need
    998                     // to go broad on the matches. Future work can hopefully make name and publisher operate more as a unit,
    999                     // but for now we have to search for the cartesian of these...
   1000                     auto names = package->GetMultiProperty(PackageMultiProperty::NormalizedName);
   1001                     auto publishers = package->GetMultiProperty(PackageMultiProperty::NormalizedPublisher);
   1002 
   1003                     for (const auto& name : names)
   1004                     {
   1005                         for (const auto& publisher : publishers)
   1006                         {
   1007                             AddIfNotPresent(SystemReferenceString{
   1008                                 PackageMatchField::NormalizedNameAndPublisher,
   1009                                 name,
   1010                                 publisher });
   1011                         }
   1012                     }
   1013                 }
   1014             };
   1015 
   1016             // For a given package, prepares the results for it.
   1017             PackageData GetSystemReferenceStrings(IPackage* package)
   1018             {
   1019                 PackageData result;
   1020                 result.AddSystemReferenceStrings(package);
   1021                 return result;
   1022             }
   1023 
   1024             // Check for a package already in the result that should have been correlated already.
   1025             // If we find one, see if we should upgrade it's match criteria.
   1026             // If we don't, return package data for further use.
   1027             //     downloadManifests: when creating system reference strings, also download manifests to get more data.
   1028             std::optional<PackageData> CheckForExistingResultFromAvailablePackageMatch(const ResultMatch& availableMatch, bool downloadManifests)
   1029             {
   1030                 std::shared_ptr<IPackage> availablePackage = OnlyAvailable(availableMatch.Package);
   1031 
   1032                 for (auto& match : Matches)
   1033                 {
   1034                     if (match.Package->IsSameAsAnyAvailable(availablePackage.get()))
   1035                     {
   1036                         if (ResultMatchComparator{}(availableMatch, match))
   1037                         {
   1038                             match.MatchCriteria = availableMatch.MatchCriteria;
   1039                         }
   1040 
   1041                         return {};
   1042                     }
   1043                 }
   1044 
   1045                 PackageData result;
   1046                 result.AddSystemReferenceStrings(availablePackage.get());
   1047 
   1048                 if (downloadManifests)
   1049                 {
   1050                     constexpr int c_downloadManifestsLimit = 3;
   1051                     int manifestsDownloaded = 0;
   1052                     for (auto const& versionKey : availablePackage->GetVersionKeys())
   1053                     {
   1054                         auto packageVersion = availablePackage->GetVersion(versionKey);
   1055 
   1056                         auto manifest = packageVersion->GetManifest();
   1057                         result.AddSystemReferenceStringsFromManifest(manifest);
   1058                         manifestsDownloaded++;
   1059 
   1060                         if (manifestsDownloaded >= c_downloadManifestsLimit)
   1061                         {
   1062                             break;
   1063                         }
   1064                     }
   1065                 }
   1066 
   1067                 return result;
   1068             }
   1069 
   1070             // Determines if the results contain the given installed package.
   1071             bool ContainsInstalledPackage(const IPackage* installedPackage) const 
   1072             {
   1073                 for (auto& match : Matches)
   1074                 {
   1075                     if (match.Package->ContainsInstalledPackage(installedPackage))
   1076                     {
   1077                         return true;
   1078                     }
   1079                 }
   1080 
   1081                 return false;
   1082             }
   1083 
   1084             // Determines if the results contain the given installed package.
   1085             std::shared_ptr<CompositePackage> FindInstalledPackage(const IPackage* installedPackage) const
   1086             {
   1087                 for (auto& match : Matches)
   1088                 {
   1089                     if (match.Package->ContainsInstalledPackage(installedPackage))
   1090                     {
   1091                         return match.Package;
   1092                     }
   1093                 }
   1094 
   1095                 return {};
   1096             }
   1097 
   1098             // *Destructively* converts the result to the standard variant.
   1099             SearchResult ConvertToSearchResult()
   1100             {
   1101                 FoldResults();
   1102 
   1103                 SearchResult result;
   1104 
   1105                 result.Matches.reserve(Matches.size());
   1106                 for (auto& match : Matches)
   1107                 {
   1108                     result.Matches.emplace_back(std::move(match.Package), std::move(match.MatchCriteria));
   1109                 }
   1110 
   1111                 result.Truncated = Truncated;
   1112 
   1113                 result.Failures = std::move(Failures);
   1114 
   1115                 return result;
   1116             }
   1117 
   1118             bool AddFailureIfSourceNotPresent(SearchResult::Failure&& failure)
   1119             {
   1120                 auto itr = std::find_if(Failures.begin(), Failures.end(),
   1121                     [&failure](const SearchResult::Failure& present) {
   1122                         return present.SourceName == failure.SourceName;
   1123                     });
   1124 
   1125                 if (itr == Failures.end())
   1126                 {
   1127                     Failures.emplace_back(std::move(failure));
   1128                     return true;
   1129                 }
   1130 
   1131                 return false;
   1132             }
   1133 
   1134             SearchResult SearchAndHandleFailures(const Source& source, const SearchRequest& request)
   1135             {
   1136                 SearchResult result;
   1137 
   1138                 try
   1139                 {
   1140                     result = source.Search(request);
   1141                 }
   1142                 catch (...)
   1143                 {
   1144                     if (AddFailureIfSourceNotPresent({ source.GetDetails().Name, std::current_exception() }))
   1145                     {
   1146                         LOG_CAUGHT_EXCEPTION();
   1147                         AICLI_LOG(Repo, Warning, << "Failed to search source for correlation: " << source.GetDetails().Name);
   1148                     }
   1149                 }
   1150 
   1151                 // Move failures into the result
   1152                 for (SearchResult::Failure& failure : result.Failures)
   1153                 {
   1154                     AddFailureIfSourceNotPresent(std::move(failure));
   1155                 }
   1156 
   1157                 return result;
   1158             }
   1159 
   1160             // Group results in an attempt to have a single result that covers all installed versions.
   1161             // This is expected to be called immediately after the installed search portion,
   1162             // when each result will contain a single installed version and some number of available packages.
   1163             // 
   1164             // The folds that happen are:
   1165             //  1. When results have the same primary available package (the primary available package is set due to tracking data)
   1166             //  2. When a result has no primary available package, but another result does have a primary that matches one of the available
   1167             //      a. Choose the latest primary if there are multiple
   1168             //  3. When multiple results have no primary available package and share the same available package set
   1169             //      a. There are many potential additional rules that could be made here, but we will start with the simplest version.
   1170             //
   1171             // Potential improvements:
   1172             //  1. Attempting correlation of non-primary available packages to allow folding in more complex cases
   1173             //      a. For example, if installed A has {source1:package1, source2:package2} and installed B has {source1:package1}, can we
   1174             //          make sure that source1:package1 and source2:package2 are in fact "the same" to confidently say that installed A and B
   1175             //          are side by side versions.
   1176             //  2. Attempt correlation by installed data only
   1177             //      a. We can potentially detect multiple instances of the same installed item with the same correlation logic turned back on
   1178             //          the installed source.  This would allow for folding even when the package is not in any available source.
   1179             void FoldResults()
   1180             {
   1181                 // The key to uniquely identify the package in the map
   1182                 struct InstalledResultFoldKey
   1183                 {
   1184                     InstalledResultFoldKey() = default;
   1185 
   1186                     InstalledResultFoldKey(const std::shared_ptr<IPackage>& package)
   1187                     {
   1188                         std::shared_ptr<IPackageVersion> latestAvailable = package->GetLatestVersion();
   1189                         if (latestAvailable)
   1190                         {
   1191                             SourceIdentifier = latestAvailable->GetSource().GetIdentifier();
   1192                             PackageIdentifier = latestAvailable->GetProperty(PackageVersionProperty::Id);
   1193                         }
   1194                     }
   1195 
   1196                     // Hash operation
   1197                     size_t operator()(const InstalledResultFoldKey& value) const noexcept
   1198                     {
   1199                         std::hash<std::string> hashString;
   1200                         return hashString(value.SourceIdentifier) ^ (hashString(value.PackageIdentifier) << 1);
   1201                     }
   1202 
   1203                     bool operator==(const InstalledResultFoldKey& other) const noexcept
   1204                     {
   1205                         // Treat both empty as invalid and never equal
   1206                         if (SourceIdentifier.empty() && PackageIdentifier.empty())
   1207                         {
   1208                             return false;
   1209                         }
   1210 
   1211                         return SourceIdentifier == other.SourceIdentifier && PackageIdentifier == other.PackageIdentifier;
   1212                     }
   1213 
   1214                     std::string SourceIdentifier;
   1215                     std::string PackageIdentifier;
   1216                 };
   1217 
   1218                 // The data for a package in the map
   1219                 struct InstalledResultFoldData
   1220                 {
   1221                     InstalledResultFoldData() = default;
   1222                     explicit InstalledResultFoldData(size_t primaryPackageIndex) : PrimaryPackageIndex(primaryPackageIndex) {}
   1223 
   1224                     std::optional<size_t> PrimaryPackageIndex;
   1225                     std::vector<size_t> NonPrimaryPackageIndices;
   1226                 };
   1227 
   1228                 std::unordered_map<InstalledResultFoldKey, InstalledResultFoldData, InstalledResultFoldKey> foldData;
   1229 
   1230                 // Attempt to fold all primary package matches first.
   1231                 // Packages without primaries will still be indexed into the hash table.
   1232                 for (size_t i = 0; i < Matches.size(); ++i)
   1233                 {
   1234                     CompositeResultMatch& currentMatch = Matches[i];
   1235 
   1236                     // Check current match for fold target
   1237                     if (currentMatch.Package->GetPrimaryAvailablePackage())
   1238                     {
   1239                         InstalledResultFoldKey key{ currentMatch.Package->GetPrimaryAvailablePackage() };
   1240 
   1241                         auto itr = foldData.find(key);
   1242                         if (itr != foldData.end())
   1243                         {
   1244                             if (itr->second.PrimaryPackageIndex)
   1245                             {
   1246                                 Matches[itr->second.PrimaryPackageIndex.value()].Package->FoldInstalledIn(currentMatch.Package);
   1247                                 currentMatch.Package.reset();
   1248                             }
   1249                             else
   1250                             {
   1251                                 itr->second.PrimaryPackageIndex = i;
   1252                             }
   1253                         }
   1254                         else
   1255                         {
   1256                             foldData[key] = InstalledResultFoldData{ i };
   1257                         }
   1258                     }
   1259                     else
   1260                     {
   1261                         for (const auto& availablePackage : currentMatch.Package->GetAvailablePackages())
   1262                         {
   1263                             InstalledResultFoldKey key{ availablePackage };
   1264 
   1265                             auto itr = foldData.find(key);
   1266                             if (itr == foldData.end())
   1267                             {
   1268                                 itr = foldData.insert({ key, {} }).first;
   1269                             }
   1270 
   1271                             itr->second.NonPrimaryPackageIndices.emplace_back(i);
   1272                         }
   1273                     }
   1274                 }
   1275 
   1276                 // After primary matches are folded, attempt to fold results without primary matches.
   1277                 // The latest primary match will be preferred.
   1278                 for (size_t i = 0; i < Matches.size(); ++i)
   1279                 {
   1280                     CompositeResultMatch& currentMatch = Matches[i];
   1281 
   1282                     // Skip any matches that we have already folded
   1283                     if (!currentMatch.Package)
   1284                     {
   1285                         continue;
   1286                     }
   1287 
   1288                     if (!currentMatch.Package->GetPrimaryAvailablePackage())
   1289                     {
   1290                         InstalledResultFoldData* latestPrimaryAvailable = nullptr;
   1291                         std::vector<InstalledResultFoldData*> availableFoldData;
   1292 
   1293                         for (const auto& availablePackage : currentMatch.Package->GetAvailablePackages())
   1294                         {
   1295                             auto& packageFoldData = foldData.at(availablePackage);
   1296 
   1297                             if (packageFoldData.PrimaryPackageIndex)
   1298                             {
   1299                                 if (!latestPrimaryAvailable ||
   1300                                     Matches[latestPrimaryAvailable->PrimaryPackageIndex.value()].Package->GetTrackingPackageWriteTime() < Matches[packageFoldData.PrimaryPackageIndex.value()].Package->GetTrackingPackageWriteTime())
   1301                                 {
   1302                                     latestPrimaryAvailable = &packageFoldData;
   1303                                 }
   1304                             }
   1305                             else
   1306                             {
   1307                                 availableFoldData.emplace_back(&packageFoldData);
   1308                             }
   1309                         }
   1310 
   1311                         if (latestPrimaryAvailable)
   1312                         {
   1313                             Matches[latestPrimaryAvailable->PrimaryPackageIndex.value()].Package->FoldInstalledIn(currentMatch.Package);
   1314                             currentMatch.Package.reset();
   1315 
   1316                             // If the result with the primary is later, move it forward
   1317                             if (latestPrimaryAvailable->PrimaryPackageIndex.value() > i)
   1318                             {
   1319                                 currentMatch.Package = std::move(Matches[latestPrimaryAvailable->PrimaryPackageIndex.value()].Package);
   1320                                 Matches[latestPrimaryAvailable->PrimaryPackageIndex.value()].Package.reset();
   1321                                 latestPrimaryAvailable->PrimaryPackageIndex = i;
   1322                             }
   1323                             continue;
   1324                         }
   1325 
   1326                         // First, find the intersection of all results that contain all of the packages from this result.
   1327                         std::vector<size_t> candidateMatches;
   1328                         for (size_t j = 0; j < availableFoldData.size(); ++j)
   1329                         {
   1330                             InstalledResultFoldData* packageFoldData = availableFoldData[j];
   1331 
   1332                             if (j == 0)
   1333                             {
   1334                                 candidateMatches = packageFoldData->NonPrimaryPackageIndices;
   1335                             }
   1336                             else
   1337                             {
   1338                                 std::vector<size_t> temp;
   1339                                 std::set_intersection(
   1340                                     candidateMatches.begin(), candidateMatches.end(),
   1341                                     packageFoldData->NonPrimaryPackageIndices.begin(), packageFoldData->NonPrimaryPackageIndices.end(),
   1342                                     std::back_inserter(temp));
   1343                                 candidateMatches = std::move(temp);
   1344                             }
   1345                         }
   1346 
   1347                         // Now exclude both our own result and any that have a different (larger) number of available packages
   1348                         candidateMatches.erase(std::remove_if(candidateMatches.begin(), candidateMatches.end(),
   1349                             [&](size_t index) { return index == i || Matches[index].Package->GetAvailablePackages().size() != currentMatch.Package->GetAvailablePackages().size(); }),
   1350                             candidateMatches.end());
   1351 
   1352                         // All of these remaining values should be folded in to our result
   1353                         for (size_t foldTarget : candidateMatches)
   1354                         {
   1355                             currentMatch.Package->FoldInstalledIn(Matches[foldTarget].Package);
   1356                             Matches[foldTarget].Package.reset();
   1357                         }
   1358                     }
   1359                 }
   1360 
   1361                 // Get rid of the folded results; we reset the Package to indicate that it is no longer valid
   1362                 Matches.erase(std::remove_if(Matches.begin(), Matches.end(), [&](const CompositeResultMatch& match) { return !match.Package; }), Matches.end());
   1363             }
   1364 
   1365             std::vector<CompositeResultMatch> Matches;
   1366             bool Truncated = false;
   1367             std::vector<SearchResult::Failure> Failures;
   1368         };
   1369 
   1370         std::shared_ptr<ICompositePackage> GetTrackedPackageFromAvailableSource(CompositeResult& result, const Source& source, const Utility::LocIndString& identifier)
   1371         {
   1372             SearchRequest directRequest;
   1373             directRequest.Filters.emplace_back(PackageMatchField::Id, MatchType::CaseInsensitive, identifier.get());
   1374 
   1375             SearchResult directResult = result.SearchAndHandleFailures(source, directRequest);
   1376 
   1377             if (directResult.Matches.empty())
   1378             {
   1379                 AICLI_LOG(Repo, Warning, << "Did not find Id [" << identifier << "] in tracked source: " << source.GetDetails().Name);
   1380             }
   1381             else if (directResult.Matches.size() == 1)
   1382             {
   1383                 return directResult.Matches[0].Package;
   1384             }
   1385             else
   1386             {
   1387                 AICLI_LOG(Repo, Warning, << "Found multiple results for Id [" << identifier << "] in tracked source: " << source.GetDetails().Name);
   1388             }
   1389 
   1390             return {};
   1391         }
   1392     }
   1393 
   1394     using namespace anon;
   1395 
   1396     CompositeSource::CompositeSource(std::string identifier)
   1397     {
   1398         m_details.Identifier = std::move(identifier);
   1399     }
   1400 
   1401     const SourceDetails& CompositeSource::GetDetails() const
   1402     {
   1403         return m_details;
   1404     }
   1405 
   1406     const std::string& CompositeSource::GetIdentifier() const
   1407     {
   1408         return m_details.Identifier;
   1409     }
   1410 
   1411     // The composite search needs to take several steps to get results, and due to the
   1412     // potential for different information spread across multiple sources, base searches
   1413     // need to be performed in both installed and available.
   1414     //
   1415     // If an installed source is present, then the searches should only return packages
   1416     // that are installed. This means that the base searches against available sources
   1417     // will only return results where a match is found in the installed source.
   1418     SearchResult CompositeSource::Search(const SearchRequest& request) const
   1419     {
   1420         if (m_installedSource)
   1421         {
   1422             return SearchInstalled(request);
   1423         }
   1424         else
   1425         {
   1426             return SearchAvailable(request);
   1427         }
   1428     }
   1429 
   1430     void* CompositeSource::CastTo(ISourceType type)
   1431     {
   1432         if (type == SourceType)
   1433         {
   1434             return this;
   1435         }
   1436 
   1437         return nullptr;
   1438     }
   1439 
   1440     void CompositeSource::AddAvailableSource(const Source& source)
   1441     {
   1442         m_availableSources.emplace_back(source);
   1443     }
   1444 
   1445     void CompositeSource::SetInstalledSource(Source source, CompositeSearchBehavior searchBehavior)
   1446     {
   1447         m_installedSource = std::move(source);
   1448         m_searchBehavior = searchBehavior;
   1449     }
   1450 
   1451     // An installed search first finds all installed packages that match the request, then correlates with available sources.
   1452     // Next the search is performed against the available sources and correlated with the installed source. A result will only
   1453     // be added if there exists an installed package that was not found by the initial search.
   1454     // This allows for search terms to find installed packages by their available metadata, as well as the local values.
   1455     //
   1456     // Search flow:
   1457     //  Installed :: Search incoming request
   1458     //  For each result
   1459     //      For each available source
   1460     //          Tracking :: Search system references
   1461     //      If tracking found
   1462     //          Available :: Search tracking ID
   1463     //      If no available, for each available source
   1464     //          Available :: Search system references
   1465     // 
   1466     //  For each available source
   1467     //      Tracking :: Search incoming request
   1468     //      For each result
   1469     //          Installed :: Search system references
   1470     //          If found
   1471     //              Available :: Search tracking ID
   1472     //      Available :: Search incoming request
   1473     //      For each result
   1474     //          Installed :: Search system references
   1475     SearchResult CompositeSource::SearchInstalled(const SearchRequest& request) const
   1476     {
   1477         CompositeResult result;
   1478 
   1479         // If the search behavior is for AllPackages or Installed then the result can contain packages that are
   1480         // only in the Installed source, but do not have an AvailableVersion.
   1481         if (m_searchBehavior == CompositeSearchBehavior::AllPackages || m_searchBehavior == CompositeSearchBehavior::Installed)
   1482         {
   1483             // Search installed source (allow exceptions out as we own the installed source)
   1484             SearchResult installedResult = m_installedSource.Search(request);
   1485             result.Truncated = installedResult.Truncated;
   1486 
   1487             for (auto&& match : installedResult.Matches)
   1488             {
   1489                 if (!match.Package)
   1490                 {
   1491                     // Ensure that the crash from installedVersion below is not from the actual package being null.
   1492                     AICLI_LOG(Repo, Warning, << "CompositeSource: The match of the package (matched on " <<
   1493                         ToString(match.MatchCriteria.Field) << " => '" << match.MatchCriteria.Value <<
   1494                         "') was null and is being dropped from the results.");
   1495                     continue;
   1496                 }
   1497 
   1498                 std::shared_ptr<CompositePackage> compositePackage = std::make_shared<CompositePackage>(match.Package);
   1499                 auto installedPackage = compositePackage->GetInstalled();
   1500 
   1501                 if (!installedPackage)
   1502                 {
   1503                     // One would think that the installed package coming directly from our own installed source
   1504                     // would never be null, but it is sometimes. Rather than making users suffer through crashes
   1505                     // that break their entire experience, lets log a few things and then ignore this match.
   1506                     AICLI_LOG(Repo, Warning, << "CompositeSource: The installed version of the package '" <<
   1507                         match.Package->GetProperty(PackageProperty::Id) << "' was null and is being dropped from the results.");
   1508                     continue;
   1509                 }
   1510 
   1511                 auto installedPackageData = result.GetSystemReferenceStrings(installedPackage.get());
   1512 
   1513                 // Create a search request to run against all available sources
   1514                 if (!installedPackageData.SystemReferenceStrings.empty())
   1515                 {
   1516                     SearchRequest systemReferenceSearch = installedPackageData.CreateInclusionsSearchRequest(SearchPurpose::CorrelationToAvailable);
   1517                     AICLI_LOG(Repo, Verbose, << "Finding available package from installed package using system reference search: " << systemReferenceSearch.ToString());
   1518 
   1519                     // Search sources and add to result
   1520                     for (const auto& source : m_availableSources)
   1521                     {
   1522                         AICLI_LOG(Repo, Verbose, << " ... searching source: " << source.GetDetails().Name << " [" << source.GetIdentifier() << ']');
   1523 
   1524                         // Find the tracking result with the latest timestamp.
   1525                         auto trackingCatalog = source.GetTrackingCatalog();
   1526                         SearchResult trackingResult = trackingCatalog.Search(systemReferenceSearch);
   1527 
   1528                         std::shared_ptr<IPackage> trackingPackage;
   1529                         std::chrono::system_clock::time_point trackingPackageTime;
   1530                         bool trackingSet = false;
   1531 
   1532                         for (const auto& trackingMatch : trackingResult.Matches)
   1533                         {
   1534                             auto candidateTime = GetLatestTrackingWriteTime(OnlyAvailable(trackingMatch.Package));
   1535 
   1536                             if (!trackingPackage || candidateTime > trackingPackageTime)
   1537                             {
   1538                                 trackingPackage = OnlyAvailable(trackingMatch.Package);
   1539                                 trackingPackageTime = candidateTime;
   1540                             }
   1541                         }
   1542 
   1543                         if (trackingPackage && trackingPackageTime > compositePackage->GetTrackingPackageWriteTime())
   1544                         {
   1545                             AICLI_LOG(Repo, Verbose, << " ... setting latest tracking package to: " << trackingPackage->GetProperty(PackageProperty::Id));
   1546                             compositePackage->SetTracking(source, trackingPackage, trackingPackageTime);
   1547                             trackingSet = true;
   1548                         }
   1549 
   1550                         // Attempt to correlate local packages against this source if supported.
   1551                         SearchResult availableResult;
   1552                         if (source.GetDetails().SupportInstalledSearchCorrelation)
   1553                         {
   1554                             availableResult = result.SearchAndHandleFailures(source, systemReferenceSearch);
   1555                         }
   1556 
   1557                         auto availablePackage = GetMatchingPackage(availableResult.Matches,
   1558                             [&]() {
   1559                                 AICLI_LOG(Repo, Info,
   1560                                 << "Found multiple matches for installed package [" << installedPackage->GetProperty(PackageProperty::Id) <<
   1561                                 "] in source [" << source.GetIdentifier() << "] when searching for [" << systemReferenceSearch.ToString() << "]");
   1562                             }, [&] {
   1563                                 AICLI_LOG(Repo, Warning, << "  Appropriate available package could not be determined");
   1564                             });
   1565 
   1566                         if (trackingPackage)
   1567                         {
   1568                             auto trackingIdentifier = trackingPackage->GetProperty(PackageProperty::Id);
   1569 
   1570                             // We always want to take the available search result if it exists as the package may have been updated.
   1571                             if (availablePackage)
   1572                             {
   1573                                 auto availableIdentifier = availablePackage->GetProperty(PackageProperty::Id);
   1574                                 if (!Utility::ICUCaseInsensitiveEquals(availableIdentifier, trackingIdentifier))
   1575                                 {
   1576                                     AICLI_LOG(Repo, Verbose, << " ... overriding tracking package (" << trackingIdentifier << ") with available package (" << availableIdentifier << ")");
   1577                                 }
   1578                             }
   1579                             else
   1580                             {
   1581                                 AICLI_LOG(Repo, Verbose, << " ... using tracking package: " << trackingIdentifier);
   1582                                 availablePackage = GetTrackedPackageFromAvailableSource(result, source, trackingIdentifier);
   1583                             }
   1584                         }
   1585 
   1586                         if (availablePackage)
   1587                         {
   1588                             AICLI_LOG(Repo, Verbose, << " ... adding available package: " << availablePackage->GetProperty(PackageProperty::Id));
   1589                             compositePackage->AddAvailablePackage(availablePackage, trackingSet);
   1590                         }
   1591                     }
   1592                 }
   1593 
   1594                 // Move the installed result into the composite result
   1595                 result.Matches.emplace_back(std::move(compositePackage), std::move(match.MatchCriteria));
   1596             }
   1597 
   1598             // Optimization for the "everything installed" case, no need to allow for reverse correlations
   1599             if (request.IsForEverything() && m_searchBehavior == CompositeSearchBehavior::Installed)
   1600             {
   1601                 return result.ConvertToSearchResult();
   1602             }
   1603         }
   1604 
   1605         // Search available sources
   1606         for (const auto& source : m_availableSources)
   1607         {
   1608             auto trackingCatalog = source.GetTrackingCatalog();
   1609 
   1610             SearchResult availableResult = result.SearchAndHandleFailures(source, request);
   1611             bool downloadManifests = source.QueryFeatureFlag(SourceFeatureFlag::ManifestMayContainAdditionalSystemReferenceStrings);
   1612 
   1613             for (auto&& match : availableResult.Matches)
   1614             {
   1615                 // Check for the package already in the result.
   1616                 // In cases that PackageData will be created, also download manifests for system reference strings
   1617                 // when search result is small (currently limiting to 1).
   1618                 auto packageData = result.CheckForExistingResultFromAvailablePackageMatch(match, downloadManifests && availableResult.Matches.size() == 1);
   1619 
   1620                 // If found existing package in the result, continue
   1621                 if (!packageData)
   1622                 {
   1623                     continue;
   1624                 }
   1625 
   1626                 // Use data from the tracking catalog as it can potentially get better correlations
   1627                 auto trackingPackage = packageData->AddSystemReferenceStringsFromTrackingPackage(trackingCatalog, match.Package->GetProperty(PackageProperty::Id), source.GetDetails().Name);
   1628 
   1629                 // If no package was found that was already in the results, do a correlation lookup with the installed
   1630                 // source to create a new composite package entry if we find any packages there.
   1631                 bool foundInstalledMatch = false;
   1632                 if (!packageData->SystemReferenceStrings.empty())
   1633                 {
   1634                     // Create a search request to run against the installed source
   1635                     SearchRequest systemReferenceSearch = packageData->CreateInclusionsSearchRequest(SearchPurpose::CorrelationToInstalled);
   1636 
   1637                     AICLI_LOG(Repo, Verbose, << "Finding installed package from available package using system reference search: " << systemReferenceSearch.ToString());
   1638                     // Correlate against installed (allow exceptions out as we own the installed source)
   1639                     SearchResult installedCrossRef = m_installedSource.Search(systemReferenceSearch);
   1640 
   1641                     for (const auto& installedMatch : installedCrossRef.Matches)
   1642                     {
   1643                         if (!IsStrongMatchField(installedMatch.MatchCriteria.Field))
   1644                         {
   1645                             // For weak correlations, do an installed -> available check to ensure that there are no other strong correlations.
   1646                             SearchResult correlationConfirmation;
   1647                             if (source.GetDetails().SupportInstalledSearchCorrelation)
   1648                             {
   1649                                 correlationConfirmation = result.SearchAndHandleFailures(source, result.GetSystemReferenceStrings(installedMatch.Package->GetInstalled().get()).CreateInclusionsSearchRequest(SearchPurpose::CorrelationToAvailable));
   1650                             }
   1651 
   1652                             if (correlationConfirmation.Matches.empty())
   1653                             {
   1654                                 // We probably made the correlation due to tracking data, keep it.
   1655                             }
   1656                             else if (correlationConfirmation.Matches.size() > 1)
   1657                             {
   1658                                 // There is contention for the correlation.
   1659                                 AICLI_LOG(Repo, Verbose, << " ... installed package [" << installedMatch.Package->GetProperty(PackageProperty::Id) <<
   1660                                     "] had multiple correlations and is being ignored as a match for [" << match.Package->GetProperty(PackageProperty::Id) << "]");
   1661                                 continue;
   1662                             }
   1663                             else if (!OnlyAvailable(correlationConfirmation.Matches[0].Package)->IsSame(OnlyAvailable(match.Package).get()))
   1664                             {
   1665                                 // The only correlation is not to the current package.
   1666                                 AICLI_LOG(Repo, Verbose, << " ... installed package [" << installedMatch.Package->GetProperty(PackageProperty::Id) <<
   1667                                     "] was found through available package [" << match.Package->GetProperty(PackageProperty::Id) << "], but only correlated to [" <<
   1668                                     correlationConfirmation.Matches[0].Package->GetProperty(PackageProperty::Id) << "] and is being ignored");
   1669                                 continue;
   1670                             }
   1671                         }
   1672 
   1673                         // Now that we know we need to add this available package, determine how exactly
   1674                         std::shared_ptr<CompositePackage> resultPackage = result.FindInstalledPackage(installedMatch.Package->GetInstalled().get());
   1675 
   1676                         if (resultPackage)
   1677                         {
   1678                             // Check for a package from the same source already present on the result package.
   1679                             bool foundSameSource = false;
   1680 
   1681                             for (const auto& availablePackage : resultPackage->GetAvailablePackages())
   1682                             {
   1683                                 if (availablePackage->GetSource() == source)
   1684                                 {
   1685                                     // TODO: May need to add more data so that we can choose the proper correlation, but it may also be very difficult to get through
   1686                                     //       the gauntlet of other checks and arrive in this situation.
   1687                                     AICLI_LOG(Repo, Verbose, << " ... found [" << availablePackage->GetProperty(PackageProperty::Id) <<
   1688                                         "] already correlated to [" << installedMatch.Package->GetProperty(PackageProperty::Id) << "] from the same source [" <<
   1689                                         source.GetDetails().Name << "] as [" << match.Package->GetProperty(PackageProperty::Id) << "]; ignoring the second correlation");
   1690                                     foundSameSource = true;
   1691                                 }
   1692                             }
   1693 
   1694                             if (foundSameSource)
   1695                             {
   1696                                 continue;
   1697                             }
   1698                         }
   1699                         else
   1700                         {
   1701                             result.Matches.emplace_back(std::make_shared<CompositePackage>(installedMatch.Package), match.MatchCriteria);
   1702                             resultPackage = result.Matches.back().Package;
   1703                         }
   1704 
   1705                         bool setPrimary = false;
   1706                         if (trackingPackage)
   1707                         {
   1708                             auto trackingPackageTime = GetLatestTrackingWriteTime(trackingPackage);
   1709 
   1710                             if (trackingPackageTime > resultPackage->GetTrackingPackageWriteTime())
   1711                             {
   1712                                 resultPackage->SetTracking(source, std::move(trackingPackage), trackingPackageTime);
   1713                                 setPrimary = true;
   1714                             }
   1715                         }
   1716 
   1717                         resultPackage->AddAvailablePackage(std::move(match.Package), setPrimary);
   1718 
   1719                         foundInstalledMatch = true;
   1720                     }
   1721                 }
   1722 
   1723                 // If there was no correlation for this package, add it without one.
   1724                 if ((m_searchBehavior == CompositeSearchBehavior::AllPackages || m_searchBehavior == CompositeSearchBehavior::AvailablePackages) && !foundInstalledMatch)
   1725                 {
   1726                     result.Matches.emplace_back(std::make_shared<CompositePackage>(std::shared_ptr<ICompositePackage>{}, std::move(match.Package)), match.MatchCriteria);
   1727                 }
   1728             }
   1729         }
   1730 
   1731         SortResultMatches(result.Matches);
   1732 
   1733         if (request.MaximumResults > 0 && result.Matches.size() > request.MaximumResults)
   1734         {
   1735             result.Truncated = true;
   1736             result.Matches.erase(result.Matches.begin() + request.MaximumResults, result.Matches.end());
   1737         }
   1738 
   1739         return result.ConvertToSearchResult();
   1740     }
   1741 
   1742     // An available search goes through each source, searching individually and then sorting the full result set.
   1743     SearchResult CompositeSource::SearchAvailable(const SearchRequest& request) const
   1744     {
   1745         SearchResult result;
   1746 
   1747         // Search available sources
   1748         for (const auto& source : m_availableSources)
   1749         {
   1750             SearchResult oneSourceResult;
   1751 
   1752             try
   1753             {
   1754                 oneSourceResult = source.Search(request);
   1755             }
   1756             catch (...)
   1757             {
   1758                 LOG_CAUGHT_EXCEPTION();
   1759                 AICLI_LOG(Repo, Warning, << "Failed to search source: " << source.GetDetails().Name);
   1760                 result.Failures.emplace_back(SearchResult::Failure{ source.GetDetails().Name, std::current_exception() });
   1761             }
   1762 
   1763             // Move into the single result
   1764             std::move(oneSourceResult.Matches.begin(), oneSourceResult.Matches.end(), std::back_inserter(result.Matches));
   1765             std::move(oneSourceResult.Failures.begin(), oneSourceResult.Failures.end(), std::back_inserter(result.Failures));
   1766         }
   1767 
   1768         SortResultMatches(result.Matches);
   1769 
   1770         if (request.MaximumResults > 0 && result.Matches.size() > request.MaximumResults)
   1771         {
   1772             result.Truncated = true;
   1773             result.Matches.erase(result.Matches.begin() + request.MaximumResults, result.Matches.end());
   1774         }
   1775 
   1776         return result;
   1777     }
   1778 }