CompositeSource.cpp (82144B)
1 // Copyright (c) Microsoft Corporation. 2 // Licensed under the MIT License. 3 #include "pch.h" 4 #include "CompositeSource.h" 5 #include <winget/ExperimentalFeature.h> 6 7 using namespace AppInstaller::Settings; 8 9 namespace AppInstaller::Repository 10 { 11 using namespace std::string_view_literals; 12 13 namespace anon 14 { 15 Utility::VersionAndChannel GetVACFromVersion(IPackageVersion* packageVersion) 16 { 17 return { 18 Utility::Version(packageVersion->GetProperty(PackageVersionProperty::Version)), 19 Utility::Channel(packageVersion->GetProperty(PackageVersionProperty::Channel)) 20 }; 21 } 22 23 // Returns true for fields that provide a strong match; one that is not based on a heuristic. 24 bool IsStrongMatchField(PackageMatchField field) 25 { 26 switch (field) 27 { 28 case AppInstaller::Repository::PackageMatchField::PackageFamilyName: 29 case AppInstaller::Repository::PackageMatchField::ProductCode: 30 case AppInstaller::Repository::PackageMatchField::UpgradeCode: 31 return true; 32 } 33 34 return false; 35 } 36 37 // Gets the only available package from the composite, ensuring this fact in test contexts. 38 std::shared_ptr<IPackage> OnlyAvailable(const std::shared_ptr<ICompositePackage>& composite) 39 { 40 std::vector<std::shared_ptr<IPackage>> availablePackages = composite->GetAvailable(); 41 42 #ifndef AICLI_DISABLE_TEST_HOOKS 43 THROW_HR_IF(E_UNEXPECTED, availablePackages.size() != 1); 44 #endif 45 46 return std::move(availablePackages.front()); 47 } 48 49 // Move returns if there is only one package in the matches that is strong; otherwise returns an empty value. 50 std::shared_ptr<ICompositePackage> FindOnlyStrongMatchFieldResult(std::vector<ResultMatch>& matches) 51 { 52 std::shared_ptr<ICompositePackage> result; 53 54 for (auto&& match : matches) 55 { 56 AICLI_LOG(Repo, Info, << " Checking match with package id: " << match.Package->GetProperty(PackageProperty::Id)); 57 58 if (IsStrongMatchField(match.MatchCriteria.Field)) 59 { 60 if (!result) 61 { 62 result = std::move(match.Package); 63 } 64 else 65 { 66 AICLI_LOG(Repo, Info, << " Found multiple packages with strong match fields"); 67 result.reset(); 68 break; 69 } 70 } 71 } 72 73 return result; 74 } 75 76 // Gets a single matching package from the results 77 template <typename MultipleIntro, typename Indeterminate> 78 std::shared_ptr<ICompositePackage> GetMatchingPackage(std::vector<ResultMatch>& matches, MultipleIntro&& multipleIntro, Indeterminate&& indeterminate) 79 { 80 if (matches.empty()) 81 { 82 return {}; 83 } 84 else if (matches.size() == 1) 85 { 86 return std::move(matches[0].Package); 87 } 88 else 89 { 90 multipleIntro(); 91 92 auto result = FindOnlyStrongMatchFieldResult(matches); 93 94 if (!result) 95 { 96 indeterminate(); 97 } 98 99 return result; 100 } 101 } 102 103 // For a given package from a tracking catalog, get the latest write time. 104 // Look at all versions rather than just the latest to account for the potential of downgrading. 105 std::chrono::system_clock::time_point GetLatestTrackingWriteTime( 106 const std::shared_ptr<IPackage>& trackingPackage) 107 { 108 std::chrono::system_clock::time_point result{}; 109 110 for (const auto& key : trackingPackage->GetVersionKeys()) 111 { 112 auto version = trackingPackage->GetVersion(key); 113 if (version) 114 { 115 auto metadata = version->GetMetadata(); 116 auto itr = metadata.find(PackageVersionMetadata::TrackingWriteTime); 117 if (itr != metadata.end()) 118 { 119 std::int64_t unixEpoch = 0; 120 try 121 { 122 unixEpoch = std::stoll(itr->second); 123 } 124 CATCH_LOG(); 125 126 std::chrono::system_clock::time_point versionTime = Utility::ConvertUnixEpochToSystemClock(unixEpoch); 127 128 if (versionTime > result) 129 { 130 result = versionTime; 131 } 132 } 133 } 134 } 135 136 return result; 137 } 138 139 // An installed package's version reported in ARP does not necessarily match the versions used for the manifest. 140 // This function uses the data in the manifest to map the installed version string to the version used by the manifest. 141 // 142 // TODO: Note: Currently this function assumes the all versions in the available package is from one source. 143 // Even though a composite package can have available packages from multiple sources, we only call this function 144 // for the default (first) available package. If we ever need to consider other sources, this function needs to be revisited. 145 std::string GetMappedInstalledVersion(const std::string& installedVersion, const std::shared_ptr<IPackage>& availablePackage) 146 { 147 // Perform an initial check to see if the latest version has a mapping; if it does not, do not attempt any more. 148 auto latestVersion = availablePackage->GetLatestVersion(); 149 if (latestVersion) 150 { 151 auto version = latestVersion->GetProperty(PackageVersionProperty::Version); 152 auto arpMinVersion = latestVersion->GetProperty(PackageVersionProperty::ArpMinVersion); 153 auto arpMaxVersion = latestVersion->GetProperty(PackageVersionProperty::ArpMaxVersion); 154 155 if ((arpMinVersion.empty() || arpMinVersion == version) && (arpMaxVersion.empty() || arpMaxVersion == version)) 156 { 157 return installedVersion; 158 } 159 } 160 161 // Stores raw versions value strings to run a preliminary check whether version mapping is needed. 162 std::vector<std::tuple<std::string, std::string, std::string>> rawVersionValues; 163 auto versionKeys = availablePackage->GetVersionKeys(); 164 bool shouldTryPerformMapping = false; 165 166 for (auto const& versionKey : versionKeys) 167 { 168 auto availableVersion = availablePackage->GetVersion(versionKey); 169 std::string arpMinVersion = availableVersion->GetProperty(PackageVersionProperty::ArpMinVersion); 170 std::string arpMaxVersion = availableVersion->GetProperty(PackageVersionProperty::ArpMaxVersion); 171 172 if (!arpMinVersion.empty() && !arpMaxVersion.empty()) 173 { 174 std::string manifestVersion = versionKey.Version; 175 176 if (!shouldTryPerformMapping && (arpMinVersion != manifestVersion || arpMaxVersion != manifestVersion)) 177 { 178 shouldTryPerformMapping = true; 179 } 180 181 rawVersionValues.emplace_back(std::make_tuple(std::move(manifestVersion), std::move(arpMinVersion), std::move(arpMaxVersion))); 182 } 183 } 184 185 if (!shouldTryPerformMapping) 186 { 187 return installedVersion; 188 } 189 190 // Construct a map between manifest version and arp version range. The map is ordered in descending by package version. 191 std::vector<std::pair<Utility::Version, Utility::VersionRange>> arpVersionMap; 192 193 for (auto& tuple : rawVersionValues) 194 { 195 auto&& [manifestVersion, arpMinVersion, arpMaxVersion] = std::move(tuple); 196 Utility::VersionRange arpVersionRange{ Utility::Version(std::move(arpMinVersion)), Utility::Version(std::move(arpMaxVersion)) }; 197 Utility::Version manifestVer{ std::move(manifestVersion) }; 198 // Skip mapping to unknown version 199 if (!manifestVer.IsUnknown()) 200 { 201 arpVersionMap.emplace_back(std::make_pair(std::move(manifestVer), std::move(arpVersionRange))); 202 } 203 } 204 205 // Go through the arp version map and determine what mapping should be performed. 206 // shouldPerformMapping is true when at least 1 arp version range is different from the package version. 207 bool shouldPerformMapping = false; 208 bool isArpVersionRangeInDescendingOrder = true; 209 const Utility::VersionRange* previousVersionRange = nullptr; 210 211 for (auto const& pair : arpVersionMap) 212 { 213 // If arp version range is not same as package version, should perform mapping 214 // This check is still needed to account for 1.0 == 1.0.0 cases 215 if (!shouldPerformMapping && !pair.second.IsSameAsSingleVersion(pair.first)) 216 { 217 shouldPerformMapping = true; 218 } 219 220 if (!previousVersionRange) 221 { 222 // This is the first non empty arp version range 223 previousVersionRange = &pair.second; 224 } 225 else if (isArpVersionRangeInDescendingOrder) 226 { 227 // The arp version range should be less than previous range 228 if (pair.second < *previousVersionRange) 229 { 230 previousVersionRange = &pair.second; 231 } 232 else 233 { 234 isArpVersionRangeInDescendingOrder = false; 235 } 236 } 237 } 238 239 // Now perform arp version mapping 240 if (shouldPerformMapping) 241 { 242 Utility::Version installed{ installedVersion }; 243 for (auto const& pair : arpVersionMap) 244 { 245 // If the installed version is in the arp version range 246 if (pair.second.ContainsVersion(installed)) 247 { 248 return pair.first.ToString(); 249 } 250 } 251 252 // At this point, no mapping found. Perform approximate mapping if applicable. 253 // We'll start from end of the vector because we try to find closest less than version if possible. 254 if (isArpVersionRangeInDescendingOrder) 255 { 256 const Utility::Version* lastGreaterThanVersion = nullptr; 257 auto it = arpVersionMap.rbegin(); 258 while (it != arpVersionMap.rend()) 259 { 260 const auto& pair = *it; 261 if (installed < pair.second.GetMinVersion()) 262 { 263 return Utility::Version{ pair.first, Utility::Version::ApproximateComparator::LessThan }.ToString(); 264 } 265 else 266 { 267 lastGreaterThanVersion = &pair.first; 268 } 269 270 it++; 271 } 272 273 // No approximate less than version found, approximate greater than version will be returned. 274 if (lastGreaterThanVersion) 275 { 276 return Utility::Version{ *lastGreaterThanVersion, Utility::Version::ApproximateComparator::GreaterThan }.ToString(); 277 } 278 } 279 } 280 281 // return the input installed version if no mapping is performed or found. 282 return installedVersion; 283 } 284 285 // A composite package installed version that allows us to override the source or the version. 286 struct CompositeInstalledVersion : public IPackageVersion 287 { 288 CompositeInstalledVersion(std::shared_ptr<IPackageVersion> baseInstalledVersion, Source trackingSource, std::shared_ptr<IPackageVersion> trackingPackageVersion, std::string overrideVersion = {}) : 289 m_baseInstalledVersion(std::move(baseInstalledVersion)), m_trackingSource(std::move(trackingSource)), m_trackingPackageVersion(std::move(trackingPackageVersion)), m_overrideVersion(std::move(overrideVersion)) 290 {} 291 292 Utility::LocIndString GetProperty(PackageVersionProperty property) const override 293 { 294 // If there is an override version, use it. 295 if (property == PackageVersionProperty::Version && !m_overrideVersion.empty()) 296 { 297 return Utility::LocIndString{ m_overrideVersion }; 298 } 299 300 return m_baseInstalledVersion->GetProperty(property); 301 } 302 303 std::vector<Utility::LocIndString> GetMultiProperty(PackageVersionMultiProperty property) const override 304 { 305 return m_baseInstalledVersion->GetMultiProperty(property); 306 } 307 308 Manifest::Manifest GetManifest() override 309 { 310 return m_baseInstalledVersion->GetManifest(); 311 } 312 313 Source GetSource() const override 314 { 315 // If there is a tracking source, use it instead to indicate that it came from there. 316 if (m_trackingSource) 317 { 318 return m_trackingSource; 319 } 320 321 return m_baseInstalledVersion->GetSource(); 322 } 323 324 Metadata GetMetadata() const override 325 { 326 auto result = m_baseInstalledVersion->GetMetadata(); 327 328 // Populate metadata from tracking package version if not present in base installed version. 329 if (m_trackingPackageVersion) 330 { 331 auto trackingMetadata = m_trackingPackageVersion->GetMetadata(); 332 for (auto metadataItem : { PackageVersionMetadata::InstalledArchitecture, PackageVersionMetadata::InstalledLocale, 333 PackageVersionMetadata::UserIntentArchitecture, PackageVersionMetadata::UserIntentLocale, PackageVersionMetadata::PinnedState }) 334 { 335 auto itr = trackingMetadata.find(metadataItem); 336 auto existingItr = result.find(metadataItem); 337 if (itr != trackingMetadata.end() && existingItr == result.end()) 338 { 339 result[metadataItem] = itr->second; 340 } 341 } 342 } 343 344 return result; 345 } 346 347 private: 348 std::shared_ptr<IPackageVersion> m_baseInstalledVersion; 349 Source m_trackingSource; 350 std::string m_overrideVersion; 351 std::shared_ptr<IPackageVersion> m_trackingPackageVersion; 352 }; 353 354 // An IPackage for the installed package of a CompositePackage. 355 struct CompositeInstalledPackage : public IPackage 356 { 357 static constexpr IPackageType PackageType = IPackageType::CompositeInstalledPackage; 358 359 CompositeInstalledPackage(std::shared_ptr<IPackage> package) 360 { 361 AddPackageAndVersionKeyData(std::move(package)); 362 } 363 364 Utility::LocIndString GetProperty(PackageProperty property) const override 365 { 366 THROW_HR_IF(E_UNEXPECTED, m_packages.empty() || m_versionKeyData.empty()); 367 368 // Use the highest version for package properties 369 return m_packages[m_versionKeyData[0].PackageIndex]->GetProperty(property); 370 } 371 372 std::vector<Utility::LocIndString> GetMultiProperty(PackageMultiProperty property) const override 373 { 374 std::vector<Utility::LocIndString> result; 375 376 for (const auto& package : m_packages) 377 { 378 for (auto&& string : package->GetMultiProperty(property)) 379 { 380 auto itr = std::lower_bound(result.begin(), result.end(), string); 381 382 if (itr == result.end() || *itr != string) 383 { 384 result.emplace(itr, std::move(string)); 385 } 386 } 387 } 388 389 return result; 390 } 391 392 std::vector<PackageVersionKey> GetVersionKeys() const override 393 { 394 return { m_versionKeyData.begin(), m_versionKeyData.end() }; 395 } 396 397 std::shared_ptr<IPackageVersion> GetVersion(const PackageVersionKey& versionKey) const override 398 { 399 std::shared_ptr<IPackageVersion> installedVersion; 400 std::string overrideVersion; 401 402 for (const VersionKeyData& key : m_versionKeyData) 403 { 404 if (key.IsMatch(versionKey)) 405 { 406 installedVersion = key.InstalledVersion; 407 overrideVersion = key.Version; 408 break; 409 } 410 } 411 412 if (installedVersion) 413 { 414 // Get the appropriate tracking version or latest if it is not found. 415 // The tracking package uses the mapped version. 416 std::shared_ptr<IPackageVersion> trackingPackageVersion; 417 if (m_trackingPackage) 418 { 419 // Remove our use of the package id as source 420 PackageVersionKey versionKey_NoSource = versionKey; 421 versionKey_NoSource.SourceId.clear(); 422 423 trackingPackageVersion = m_trackingPackage->GetVersion(versionKey_NoSource); 424 425 if (!trackingPackageVersion) 426 { 427 trackingPackageVersion = m_trackingPackage->GetLatestVersion(); 428 } 429 } 430 431 return std::make_shared<CompositeInstalledVersion>(std::move(installedVersion), m_trackingSource, std::move(trackingPackageVersion), std::move(overrideVersion)); 432 } 433 434 return nullptr; 435 } 436 437 std::shared_ptr<IPackageVersion> GetLatestVersion() const override 438 { 439 return GetVersion({}); 440 } 441 442 Source GetSource() const override 443 { 444 // If there is a tracking source, use it instead to indicate that it came from there. 445 // Otherwise, all of the installed packages should be from the same source. 446 return m_trackingSource ? m_trackingSource : m_packages[0]->GetSource(); 447 } 448 449 bool IsSame(const IPackage* other) const override 450 { 451 const CompositeInstalledPackage* otherPackage = PackageCast<const CompositeInstalledPackage*>(other); 452 453 if (otherPackage) 454 { 455 if (m_packages.size() != otherPackage->m_packages.size()) 456 { 457 return false; 458 } 459 460 for (const auto& subPackage : m_packages) 461 { 462 bool foundSame = false; 463 464 for (const auto& otherSubPackage : otherPackage->m_packages) 465 { 466 if (subPackage->IsSame(otherSubPackage.get())) 467 { 468 foundSame = true; 469 break; 470 } 471 } 472 473 if (!foundSame) 474 { 475 return false; 476 } 477 } 478 479 return true; 480 } 481 482 return false; 483 } 484 485 const void* CastTo(IPackageType type) const override 486 { 487 if (type == PackageType) 488 { 489 return this; 490 } 491 492 return nullptr; 493 } 494 495 void SetTracking( 496 Source trackingSource, 497 std::shared_ptr<IPackage> trackingPackage, 498 std::chrono::system_clock::time_point trackingWriteTime) 499 { 500 m_trackingSource = std::move(trackingSource); 501 m_trackingPackage = std::move(trackingPackage); 502 m_trackingWriteTime = trackingWriteTime; 503 } 504 505 Source GetTrackingSource() const 506 { 507 return m_trackingSource; 508 } 509 510 const std::shared_ptr<IPackage>& GetTrackingPackage() const 511 { 512 return m_trackingPackage; 513 } 514 515 std::chrono::system_clock::time_point GetTrackingPackageWriteTime() const 516 { 517 return m_trackingWriteTime; 518 } 519 520 bool ContainsInstalledPackage(const IPackage* installedPackage) const 521 { 522 for (const auto& package : m_packages) 523 { 524 if (package->IsSame(installedPackage)) 525 { 526 return true; 527 } 528 } 529 530 return false; 531 } 532 533 void FoldInstalledIn(const std::shared_ptr<CompositeInstalledPackage>& other) 534 { 535 for (const auto& package : other->m_packages) 536 { 537 AddPackageAndVersionKeyData(package); 538 } 539 } 540 541 // Set a version that will override the version string from the installed package 542 void SetOverrideInstalledVersion(const std::shared_ptr<IPackage>& availablePackage) 543 { 544 if (availablePackage) 545 { 546 m_availablePackageVersionOverride = availablePackage; 547 548 for (auto& key : m_versionKeyData) 549 { 550 if (Manifest::DoesInstallerTypeSupportArpVersionRange(key.InstalledType)) 551 { 552 key.Version = GetMappedInstalledVersion(key.InstalledVersion->GetProperty(PackageVersionProperty::Version), availablePackage); 553 } 554 } 555 } 556 } 557 558 bool IsEmpty() const 559 { 560 return m_versionKeyData.empty(); 561 } 562 563 private: 564 // Contains information about all of the version keys. 565 // We use the `SourceId` field to store the installed package identifier so that we can disambiguate keys is they have the same version. 566 struct VersionKeyData : public PackageVersionKey 567 { 568 size_t PackageIndex; 569 std::shared_ptr<IPackageVersion> InstalledVersion; 570 Manifest::InstallerTypeEnum InstalledType; 571 Utility::VersionAndChannel VersionAndChannel; 572 573 bool operator<(const VersionKeyData& other) const 574 { 575 return VersionAndChannel < other.VersionAndChannel; 576 } 577 }; 578 579 // Adds the package and version key data to the composite. 580 // The version keys are then sorted so that the first (index 0) in the vector has the highest version. 581 // Note that it may tied for highest version if, for instance, the same version is installed for different architectures. 582 void AddPackageAndVersionKeyData(std::shared_ptr<IPackage> package) 583 { 584 // We don't want this to happen, but it could. Rather than a crash, we will log it and move on. 585 if (!package) 586 { 587 AICLI_LOG(Repo, Verbose, << "AddPackageAndVersionKeyData called with an empty package"); 588 return; 589 } 590 591 size_t packageIndex = m_packages.size(); 592 std::string packageIdentifier = package->GetProperty(PackageProperty::Id); 593 bool versionAdded = false; 594 595 for (const auto& versionKey : package->GetVersionKeys()) 596 { 597 VersionKeyData keyData{ versionKey }; 598 599 keyData.PackageIndex = packageIndex; 600 keyData.InstalledVersion = package->GetVersion(versionKey); 601 602 if (!keyData.InstalledVersion) 603 { 604 AICLI_LOG(Repo, Verbose, << "AddPackageAndVersionKeyData: Package [" << packageIdentifier << "] did not return a version for [" << versionKey.Version << "]"); 605 continue; 606 } 607 608 // We use the `SourceId` field to store the installed package identifier so that we can disambiguate keys if they have the same version. 609 keyData.SourceId = packageIdentifier; 610 611 keyData.InstalledType = Manifest::ConvertToInstallerTypeEnum(keyData.InstalledVersion->GetMetadata()[PackageVersionMetadata::InstalledType]); 612 if (m_availablePackageVersionOverride && Manifest::DoesInstallerTypeSupportArpVersionRange(keyData.InstalledType)) 613 { 614 keyData.Version = GetMappedInstalledVersion(keyData.InstalledVersion->GetProperty(PackageVersionProperty::Version), m_availablePackageVersionOverride); 615 } 616 617 keyData.VersionAndChannel = Utility::VersionAndChannel{ keyData.Version, keyData.Channel }; 618 619 m_versionKeyData.emplace_back(std::move(keyData)); 620 versionAdded = true; 621 } 622 623 if (versionAdded) 624 { 625 m_packages.emplace_back(std::move(package)); 626 627 std::sort(m_versionKeyData.begin(), m_versionKeyData.end()); 628 } 629 } 630 631 std::vector<std::shared_ptr<IPackage>> m_packages; 632 std::vector<VersionKeyData> m_versionKeyData; 633 Source m_trackingSource; 634 std::shared_ptr<IPackage> m_trackingPackage; 635 std::chrono::system_clock::time_point m_trackingWriteTime = std::chrono::system_clock::time_point::min(); 636 std::shared_ptr<IPackage> m_availablePackageVersionOverride; 637 }; 638 639 // An ICompositePackage for the CompositeSource. 640 struct CompositePackage : public ICompositePackage 641 { 642 // The availablePackage may only contain one available package within it, as it is expected to be the output of a search on a single source. 643 CompositePackage(const std::shared_ptr<ICompositePackage>& installedPackage, const std::shared_ptr<ICompositePackage>& availablePackage = {}, bool setPrimary = false) 644 { 645 if (installedPackage) 646 { 647 m_installedPackage = std::make_shared<CompositeInstalledPackage>(installedPackage->GetInstalled()); 648 649 // If the installed package result existed, but didn't actually create any installed versions, drop it. 650 if (m_installedPackage->IsEmpty()) 651 { 652 m_installedPackage.reset(); 653 } 654 } 655 656 AddAvailablePackage(availablePackage, setPrimary); 657 } 658 659 Utility::LocIndString GetProperty(PackageProperty property) const override 660 { 661 IPackage* truth = nullptr; 662 if (m_primaryAvailablePackage) 663 { 664 truth = m_primaryAvailablePackage.get(); 665 } 666 if (!truth && !m_availablePackages.empty()) 667 { 668 truth = m_availablePackages[0].get(); 669 } 670 if (!truth) 671 { 672 truth = m_installedPackage.get(); 673 } 674 675 THROW_HR_IF(E_UNEXPECTED, !truth); 676 677 return truth->GetProperty(property); 678 } 679 680 std::shared_ptr<IPackage> GetInstalled() override 681 { 682 return m_installedPackage; 683 } 684 685 std::vector<std::shared_ptr<IPackage>> GetAvailable() override 686 { 687 return m_availablePackages; 688 } 689 690 const std::vector<std::shared_ptr<IPackage>>& GetAvailablePackages() 691 { 692 return m_availablePackages; 693 } 694 695 bool IsSameAsAnyAvailable(const IPackage* other) const 696 { 697 if (other) 698 { 699 for (const auto& availablePackage : m_availablePackages) 700 { 701 if (other->IsSame(availablePackage.get())) 702 { 703 return true; 704 } 705 } 706 } 707 708 return false; 709 } 710 711 const std::shared_ptr<CompositeInstalledPackage>& GetInstalledPackage() const 712 { 713 return m_installedPackage; 714 } 715 716 bool ContainsInstalledPackage(const IPackage* installedPackage) const 717 { 718 return m_installedPackage ? m_installedPackage->ContainsInstalledPackage(installedPackage) : false; 719 } 720 721 void AddAvailablePackage(const std::shared_ptr<ICompositePackage>& availablePackage, bool setPrimary = false) 722 { 723 if (availablePackage) 724 { 725 m_availablePackages.emplace_back(OnlyAvailable(availablePackage)); 726 727 if (setPrimary) 728 { 729 m_primaryAvailablePackage = m_availablePackages.back(); 730 } 731 732 // Set override for primary or with the first available version found 733 if (setPrimary || m_availablePackages.size() == 1) 734 { 735 TrySetOverrideInstalledVersion(m_availablePackages.back()); 736 } 737 } 738 } 739 740 std::shared_ptr<IPackage>& GetPrimaryAvailablePackage() 741 { 742 return m_primaryAvailablePackage; 743 } 744 745 Source GetTrackingSource() const 746 { 747 return m_installedPackage ? m_installedPackage->GetTrackingSource() : Source{}; 748 } 749 750 std::shared_ptr<IPackage> GetTrackingPackage() const 751 { 752 return m_installedPackage ? m_installedPackage->GetTrackingPackage() : std::shared_ptr<IPackage>{}; 753 } 754 755 std::chrono::system_clock::time_point GetTrackingPackageWriteTime() const 756 { 757 return m_installedPackage ? m_installedPackage->GetTrackingPackageWriteTime() : std::chrono::system_clock::time_point::min(); 758 } 759 760 void SetTracking( 761 Source trackingSource, 762 std::shared_ptr<IPackage> trackingPackage, 763 std::chrono::system_clock::time_point trackingWriteTime) 764 { 765 if (m_installedPackage) 766 { 767 m_installedPackage->SetTracking(std::move(trackingSource), std::move(trackingPackage), trackingWriteTime); 768 } 769 } 770 771 void FoldInstalledIn(const std::shared_ptr<CompositePackage>& other) 772 { 773 if (other->m_installedPackage) 774 { 775 if (m_installedPackage) 776 { 777 m_installedPackage->FoldInstalledIn(other->m_installedPackage); 778 } 779 else 780 { 781 m_installedPackage = other->m_installedPackage; 782 } 783 } 784 } 785 786 private: 787 // Try to set a version that will override the version string from the installed package 788 void TrySetOverrideInstalledVersion(const std::shared_ptr<IPackage>& availablePackage) 789 { 790 if (m_installedPackage && availablePackage) 791 { 792 m_installedPackage->SetOverrideInstalledVersion(availablePackage); 793 } 794 } 795 796 std::shared_ptr<CompositeInstalledPackage> m_installedPackage; 797 std::shared_ptr<IPackage> m_primaryAvailablePackage; 798 std::vector<std::shared_ptr<IPackage>> m_availablePackages; 799 }; 800 801 // The comparator compares the ResultMatch by MatchType first, then Field in a predefined order. 802 struct ResultMatchComparator 803 { 804 template <typename U, typename V> 805 bool operator() ( 806 const U& match1, 807 const V& match2) 808 { 809 if (match1.MatchCriteria.Type != match2.MatchCriteria.Type) 810 { 811 return match1.MatchCriteria.Type < match2.MatchCriteria.Type; 812 } 813 814 if (match1.MatchCriteria.Field != match2.MatchCriteria.Field) 815 { 816 return match1.MatchCriteria.Field < match2.MatchCriteria.Field; 817 } 818 819 return false; 820 } 821 }; 822 823 template <typename T> 824 void SortResultMatches(std::vector<T>& matches) 825 { 826 std::stable_sort(matches.begin(), matches.end(), ResultMatchComparator()); 827 } 828 829 // A copy of the standard match that holds a CompositePackage instead. 830 struct CompositeResultMatch 831 { 832 std::shared_ptr<CompositePackage> Package; 833 PackageMatchFilter MatchCriteria; 834 835 CompositeResultMatch(std::shared_ptr<CompositePackage> p, PackageMatchFilter f) : Package(std::move(p)), MatchCriteria(std::move(f)) {} 836 }; 837 838 // Stores data to enable correlation between installed and available packages. 839 struct CompositeResult 840 { 841 // A system reference string. 842 struct SystemReferenceString 843 { 844 SystemReferenceString(PackageMatchField field, Utility::LocIndString string) : 845 Field(field), String1(Utility::FoldCase(string)) {} 846 847 SystemReferenceString(PackageMatchField field, Utility::LocIndString string1, Utility::LocIndString string2) : 848 Field(field), String1(Utility::FoldCase(string1)), String2(Utility::FoldCase(string2)) {} 849 850 bool operator<(const SystemReferenceString& other) const 851 { 852 if (Field != other.Field) 853 { 854 return Field < other.Field; 855 } 856 857 if (String1 != other.String1) 858 { 859 return String1 < other.String1; 860 } 861 862 return String2 < other.String2; 863 } 864 865 bool operator==(const SystemReferenceString& other) const 866 { 867 return Field == other.Field && String1 == other.String1 && String2 == other.String2; 868 } 869 870 void AddToFilters( 871 std::vector<PackageMatchFilter>& filters) const 872 { 873 switch (Field) 874 { 875 case PackageMatchField::NormalizedNameAndPublisher: 876 filters.emplace_back(PackageMatchFilter(Field, MatchType::Exact, String1.get(), String2.get())); 877 break; 878 879 default: 880 filters.emplace_back(PackageMatchFilter(Field, MatchType::Exact, String1.get())); 881 } 882 } 883 884 private: 885 PackageMatchField Field; 886 Utility::LocIndString String1; 887 Utility::LocIndString String2; 888 }; 889 890 // Data relevant to correlation for a package. 891 struct PackageData 892 { 893 std::set<SystemReferenceString> SystemReferenceStrings; 894 895 void AddIfNotPresent(SystemReferenceString&& srs) 896 { 897 if (SystemReferenceStrings.find(srs) == SystemReferenceStrings.end()) 898 { 899 SystemReferenceStrings.emplace(std::move(srs)); 900 } 901 } 902 903 SearchRequest CreateInclusionsSearchRequest(SearchPurpose searchPurpose) const 904 { 905 SearchRequest result; 906 for (const auto& srs : SystemReferenceStrings) 907 { 908 srs.AddToFilters(result.Inclusions); 909 } 910 result.Purpose = searchPurpose; 911 return result; 912 } 913 914 std::shared_ptr<IPackage> AddSystemReferenceStringsFromTrackingPackage(const PackageTrackingCatalog& trackingCatalog, const Utility::LocIndString& identifier, std::string_view sourceIdentifier) 915 { 916 SearchRequest trackingRequest; 917 trackingRequest.Filters.emplace_back(PackageMatchField::Id, MatchType::CaseInsensitive, identifier.get()); 918 919 SearchResult trackingResult = trackingCatalog.Search(trackingRequest); 920 std::shared_ptr<IPackage> result; 921 922 if (trackingResult.Matches.size() == 1) 923 { 924 result = OnlyAvailable(trackingResult.Matches[0].Package); 925 AddSystemReferenceStrings(result.get()); 926 } 927 else if (trackingResult.Matches.size() > 1) 928 { 929 AICLI_LOG(Repo, Warning, << "Found " << trackingResult.Matches.size() << " results for Id [" << identifier << "] in tracking catalog for: " << sourceIdentifier); 930 } 931 932 return result; 933 } 934 935 void AddSystemReferenceStrings(IPackage* package) 936 { 937 GetSystemReferenceStrings( 938 package, 939 PackageMultiProperty::PackageFamilyName, 940 PackageMatchField::PackageFamilyName); 941 942 GetSystemReferenceStrings( 943 package, 944 PackageMultiProperty::ProductCode, 945 PackageMatchField::ProductCode); 946 947 GetSystemReferenceStrings( 948 package, 949 PackageMultiProperty::UpgradeCode, 950 PackageMatchField::UpgradeCode); 951 952 GetNameAndPublisher( 953 package); 954 } 955 956 void AddSystemReferenceStringsFromManifest(const Manifest::Manifest& manifest) 957 { 958 for (const auto& pfn : manifest.GetPackageFamilyNames()) 959 { 960 AddIfNotPresent(SystemReferenceString{ PackageMatchField::PackageFamilyName, Utility::LocIndString{ pfn } }); 961 } 962 for (const auto& productCode : manifest.GetProductCodes()) 963 { 964 AddIfNotPresent(SystemReferenceString{ PackageMatchField::ProductCode, Utility::LocIndString{ productCode } }); 965 } 966 for (const auto& upgradeCode : manifest.GetUpgradeCodes()) 967 { 968 AddIfNotPresent(SystemReferenceString{ PackageMatchField::UpgradeCode, Utility::LocIndString{ upgradeCode } }); 969 } 970 for (const auto& name : manifest.GetPackageNames()) 971 { 972 for (const auto& publisher : manifest.GetPublishers()) 973 { 974 AddIfNotPresent(SystemReferenceString{ 975 PackageMatchField::NormalizedNameAndPublisher, 976 Utility::LocIndString{ name }, 977 Utility::LocIndString{ publisher } }); 978 } 979 } 980 } 981 982 private: 983 void GetSystemReferenceStrings( 984 IPackage* package, 985 PackageMultiProperty prop, 986 PackageMatchField field) 987 { 988 for (auto&& string : package->GetMultiProperty(prop)) 989 { 990 AddIfNotPresent(SystemReferenceString{ field, std::move(string) }); 991 } 992 } 993 994 void GetNameAndPublisher( 995 IPackage* package) 996 { 997 // Unfortunately the names and publishers are unique and not tied to each other strictly, so we need 998 // to go broad on the matches. Future work can hopefully make name and publisher operate more as a unit, 999 // but for now we have to search for the cartesian of these... 1000 auto names = package->GetMultiProperty(PackageMultiProperty::NormalizedName); 1001 auto publishers = package->GetMultiProperty(PackageMultiProperty::NormalizedPublisher); 1002 1003 for (const auto& name : names) 1004 { 1005 for (const auto& publisher : publishers) 1006 { 1007 AddIfNotPresent(SystemReferenceString{ 1008 PackageMatchField::NormalizedNameAndPublisher, 1009 name, 1010 publisher }); 1011 } 1012 } 1013 } 1014 }; 1015 1016 // For a given package, prepares the results for it. 1017 PackageData GetSystemReferenceStrings(IPackage* package) 1018 { 1019 PackageData result; 1020 result.AddSystemReferenceStrings(package); 1021 return result; 1022 } 1023 1024 // Check for a package already in the result that should have been correlated already. 1025 // If we find one, see if we should upgrade it's match criteria. 1026 // If we don't, return package data for further use. 1027 // downloadManifests: when creating system reference strings, also download manifests to get more data. 1028 std::optional<PackageData> CheckForExistingResultFromAvailablePackageMatch(const ResultMatch& availableMatch, bool downloadManifests) 1029 { 1030 std::shared_ptr<IPackage> availablePackage = OnlyAvailable(availableMatch.Package); 1031 1032 for (auto& match : Matches) 1033 { 1034 if (match.Package->IsSameAsAnyAvailable(availablePackage.get())) 1035 { 1036 if (ResultMatchComparator{}(availableMatch, match)) 1037 { 1038 match.MatchCriteria = availableMatch.MatchCriteria; 1039 } 1040 1041 return {}; 1042 } 1043 } 1044 1045 PackageData result; 1046 result.AddSystemReferenceStrings(availablePackage.get()); 1047 1048 if (downloadManifests) 1049 { 1050 constexpr int c_downloadManifestsLimit = 3; 1051 int manifestsDownloaded = 0; 1052 for (auto const& versionKey : availablePackage->GetVersionKeys()) 1053 { 1054 auto packageVersion = availablePackage->GetVersion(versionKey); 1055 1056 auto manifest = packageVersion->GetManifest(); 1057 result.AddSystemReferenceStringsFromManifest(manifest); 1058 manifestsDownloaded++; 1059 1060 if (manifestsDownloaded >= c_downloadManifestsLimit) 1061 { 1062 break; 1063 } 1064 } 1065 } 1066 1067 return result; 1068 } 1069 1070 // Determines if the results contain the given installed package. 1071 bool ContainsInstalledPackage(const IPackage* installedPackage) const 1072 { 1073 for (auto& match : Matches) 1074 { 1075 if (match.Package->ContainsInstalledPackage(installedPackage)) 1076 { 1077 return true; 1078 } 1079 } 1080 1081 return false; 1082 } 1083 1084 // Determines if the results contain the given installed package. 1085 std::shared_ptr<CompositePackage> FindInstalledPackage(const IPackage* installedPackage) const 1086 { 1087 for (auto& match : Matches) 1088 { 1089 if (match.Package->ContainsInstalledPackage(installedPackage)) 1090 { 1091 return match.Package; 1092 } 1093 } 1094 1095 return {}; 1096 } 1097 1098 // *Destructively* converts the result to the standard variant. 1099 SearchResult ConvertToSearchResult() 1100 { 1101 FoldResults(); 1102 1103 SearchResult result; 1104 1105 result.Matches.reserve(Matches.size()); 1106 for (auto& match : Matches) 1107 { 1108 result.Matches.emplace_back(std::move(match.Package), std::move(match.MatchCriteria)); 1109 } 1110 1111 result.Truncated = Truncated; 1112 1113 result.Failures = std::move(Failures); 1114 1115 return result; 1116 } 1117 1118 bool AddFailureIfSourceNotPresent(SearchResult::Failure&& failure) 1119 { 1120 auto itr = std::find_if(Failures.begin(), Failures.end(), 1121 [&failure](const SearchResult::Failure& present) { 1122 return present.SourceName == failure.SourceName; 1123 }); 1124 1125 if (itr == Failures.end()) 1126 { 1127 Failures.emplace_back(std::move(failure)); 1128 return true; 1129 } 1130 1131 return false; 1132 } 1133 1134 SearchResult SearchAndHandleFailures(const Source& source, const SearchRequest& request) 1135 { 1136 SearchResult result; 1137 1138 try 1139 { 1140 result = source.Search(request); 1141 } 1142 catch (...) 1143 { 1144 if (AddFailureIfSourceNotPresent({ source.GetDetails().Name, std::current_exception() })) 1145 { 1146 LOG_CAUGHT_EXCEPTION(); 1147 AICLI_LOG(Repo, Warning, << "Failed to search source for correlation: " << source.GetDetails().Name); 1148 } 1149 } 1150 1151 // Move failures into the result 1152 for (SearchResult::Failure& failure : result.Failures) 1153 { 1154 AddFailureIfSourceNotPresent(std::move(failure)); 1155 } 1156 1157 return result; 1158 } 1159 1160 // Group results in an attempt to have a single result that covers all installed versions. 1161 // This is expected to be called immediately after the installed search portion, 1162 // when each result will contain a single installed version and some number of available packages. 1163 // 1164 // The folds that happen are: 1165 // 1. When results have the same primary available package (the primary available package is set due to tracking data) 1166 // 2. When a result has no primary available package, but another result does have a primary that matches one of the available 1167 // a. Choose the latest primary if there are multiple 1168 // 3. When multiple results have no primary available package and share the same available package set 1169 // a. There are many potential additional rules that could be made here, but we will start with the simplest version. 1170 // 1171 // Potential improvements: 1172 // 1. Attempting correlation of non-primary available packages to allow folding in more complex cases 1173 // a. For example, if installed A has {source1:package1, source2:package2} and installed B has {source1:package1}, can we 1174 // make sure that source1:package1 and source2:package2 are in fact "the same" to confidently say that installed A and B 1175 // are side by side versions. 1176 // 2. Attempt correlation by installed data only 1177 // a. We can potentially detect multiple instances of the same installed item with the same correlation logic turned back on 1178 // the installed source. This would allow for folding even when the package is not in any available source. 1179 void FoldResults() 1180 { 1181 // The key to uniquely identify the package in the map 1182 struct InstalledResultFoldKey 1183 { 1184 InstalledResultFoldKey() = default; 1185 1186 InstalledResultFoldKey(const std::shared_ptr<IPackage>& package) 1187 { 1188 std::shared_ptr<IPackageVersion> latestAvailable = package->GetLatestVersion(); 1189 if (latestAvailable) 1190 { 1191 SourceIdentifier = latestAvailable->GetSource().GetIdentifier(); 1192 PackageIdentifier = latestAvailable->GetProperty(PackageVersionProperty::Id); 1193 } 1194 } 1195 1196 // Hash operation 1197 size_t operator()(const InstalledResultFoldKey& value) const noexcept 1198 { 1199 std::hash<std::string> hashString; 1200 return hashString(value.SourceIdentifier) ^ (hashString(value.PackageIdentifier) << 1); 1201 } 1202 1203 bool operator==(const InstalledResultFoldKey& other) const noexcept 1204 { 1205 // Treat both empty as invalid and never equal 1206 if (SourceIdentifier.empty() && PackageIdentifier.empty()) 1207 { 1208 return false; 1209 } 1210 1211 return SourceIdentifier == other.SourceIdentifier && PackageIdentifier == other.PackageIdentifier; 1212 } 1213 1214 std::string SourceIdentifier; 1215 std::string PackageIdentifier; 1216 }; 1217 1218 // The data for a package in the map 1219 struct InstalledResultFoldData 1220 { 1221 InstalledResultFoldData() = default; 1222 explicit InstalledResultFoldData(size_t primaryPackageIndex) : PrimaryPackageIndex(primaryPackageIndex) {} 1223 1224 std::optional<size_t> PrimaryPackageIndex; 1225 std::vector<size_t> NonPrimaryPackageIndices; 1226 }; 1227 1228 std::unordered_map<InstalledResultFoldKey, InstalledResultFoldData, InstalledResultFoldKey> foldData; 1229 1230 // Attempt to fold all primary package matches first. 1231 // Packages without primaries will still be indexed into the hash table. 1232 for (size_t i = 0; i < Matches.size(); ++i) 1233 { 1234 CompositeResultMatch& currentMatch = Matches[i]; 1235 1236 // Check current match for fold target 1237 if (currentMatch.Package->GetPrimaryAvailablePackage()) 1238 { 1239 InstalledResultFoldKey key{ currentMatch.Package->GetPrimaryAvailablePackage() }; 1240 1241 auto itr = foldData.find(key); 1242 if (itr != foldData.end()) 1243 { 1244 if (itr->second.PrimaryPackageIndex) 1245 { 1246 Matches[itr->second.PrimaryPackageIndex.value()].Package->FoldInstalledIn(currentMatch.Package); 1247 currentMatch.Package.reset(); 1248 } 1249 else 1250 { 1251 itr->second.PrimaryPackageIndex = i; 1252 } 1253 } 1254 else 1255 { 1256 foldData[key] = InstalledResultFoldData{ i }; 1257 } 1258 } 1259 else 1260 { 1261 for (const auto& availablePackage : currentMatch.Package->GetAvailablePackages()) 1262 { 1263 InstalledResultFoldKey key{ availablePackage }; 1264 1265 auto itr = foldData.find(key); 1266 if (itr == foldData.end()) 1267 { 1268 itr = foldData.insert({ key, {} }).first; 1269 } 1270 1271 itr->second.NonPrimaryPackageIndices.emplace_back(i); 1272 } 1273 } 1274 } 1275 1276 // After primary matches are folded, attempt to fold results without primary matches. 1277 // The latest primary match will be preferred. 1278 for (size_t i = 0; i < Matches.size(); ++i) 1279 { 1280 CompositeResultMatch& currentMatch = Matches[i]; 1281 1282 // Skip any matches that we have already folded 1283 if (!currentMatch.Package) 1284 { 1285 continue; 1286 } 1287 1288 if (!currentMatch.Package->GetPrimaryAvailablePackage()) 1289 { 1290 InstalledResultFoldData* latestPrimaryAvailable = nullptr; 1291 std::vector<InstalledResultFoldData*> availableFoldData; 1292 1293 for (const auto& availablePackage : currentMatch.Package->GetAvailablePackages()) 1294 { 1295 auto& packageFoldData = foldData.at(availablePackage); 1296 1297 if (packageFoldData.PrimaryPackageIndex) 1298 { 1299 if (!latestPrimaryAvailable || 1300 Matches[latestPrimaryAvailable->PrimaryPackageIndex.value()].Package->GetTrackingPackageWriteTime() < Matches[packageFoldData.PrimaryPackageIndex.value()].Package->GetTrackingPackageWriteTime()) 1301 { 1302 latestPrimaryAvailable = &packageFoldData; 1303 } 1304 } 1305 else 1306 { 1307 availableFoldData.emplace_back(&packageFoldData); 1308 } 1309 } 1310 1311 if (latestPrimaryAvailable) 1312 { 1313 Matches[latestPrimaryAvailable->PrimaryPackageIndex.value()].Package->FoldInstalledIn(currentMatch.Package); 1314 currentMatch.Package.reset(); 1315 1316 // If the result with the primary is later, move it forward 1317 if (latestPrimaryAvailable->PrimaryPackageIndex.value() > i) 1318 { 1319 currentMatch.Package = std::move(Matches[latestPrimaryAvailable->PrimaryPackageIndex.value()].Package); 1320 Matches[latestPrimaryAvailable->PrimaryPackageIndex.value()].Package.reset(); 1321 latestPrimaryAvailable->PrimaryPackageIndex = i; 1322 } 1323 continue; 1324 } 1325 1326 // First, find the intersection of all results that contain all of the packages from this result. 1327 std::vector<size_t> candidateMatches; 1328 for (size_t j = 0; j < availableFoldData.size(); ++j) 1329 { 1330 InstalledResultFoldData* packageFoldData = availableFoldData[j]; 1331 1332 if (j == 0) 1333 { 1334 candidateMatches = packageFoldData->NonPrimaryPackageIndices; 1335 } 1336 else 1337 { 1338 std::vector<size_t> temp; 1339 std::set_intersection( 1340 candidateMatches.begin(), candidateMatches.end(), 1341 packageFoldData->NonPrimaryPackageIndices.begin(), packageFoldData->NonPrimaryPackageIndices.end(), 1342 std::back_inserter(temp)); 1343 candidateMatches = std::move(temp); 1344 } 1345 } 1346 1347 // Now exclude both our own result and any that have a different (larger) number of available packages 1348 candidateMatches.erase(std::remove_if(candidateMatches.begin(), candidateMatches.end(), 1349 [&](size_t index) { return index == i || Matches[index].Package->GetAvailablePackages().size() != currentMatch.Package->GetAvailablePackages().size(); }), 1350 candidateMatches.end()); 1351 1352 // All of these remaining values should be folded in to our result 1353 for (size_t foldTarget : candidateMatches) 1354 { 1355 currentMatch.Package->FoldInstalledIn(Matches[foldTarget].Package); 1356 Matches[foldTarget].Package.reset(); 1357 } 1358 } 1359 } 1360 1361 // Get rid of the folded results; we reset the Package to indicate that it is no longer valid 1362 Matches.erase(std::remove_if(Matches.begin(), Matches.end(), [&](const CompositeResultMatch& match) { return !match.Package; }), Matches.end()); 1363 } 1364 1365 std::vector<CompositeResultMatch> Matches; 1366 bool Truncated = false; 1367 std::vector<SearchResult::Failure> Failures; 1368 }; 1369 1370 std::shared_ptr<ICompositePackage> GetTrackedPackageFromAvailableSource(CompositeResult& result, const Source& source, const Utility::LocIndString& identifier) 1371 { 1372 SearchRequest directRequest; 1373 directRequest.Filters.emplace_back(PackageMatchField::Id, MatchType::CaseInsensitive, identifier.get()); 1374 1375 SearchResult directResult = result.SearchAndHandleFailures(source, directRequest); 1376 1377 if (directResult.Matches.empty()) 1378 { 1379 AICLI_LOG(Repo, Warning, << "Did not find Id [" << identifier << "] in tracked source: " << source.GetDetails().Name); 1380 } 1381 else if (directResult.Matches.size() == 1) 1382 { 1383 return directResult.Matches[0].Package; 1384 } 1385 else 1386 { 1387 AICLI_LOG(Repo, Warning, << "Found multiple results for Id [" << identifier << "] in tracked source: " << source.GetDetails().Name); 1388 } 1389 1390 return {}; 1391 } 1392 } 1393 1394 using namespace anon; 1395 1396 CompositeSource::CompositeSource(std::string identifier) 1397 { 1398 m_details.Identifier = std::move(identifier); 1399 } 1400 1401 const SourceDetails& CompositeSource::GetDetails() const 1402 { 1403 return m_details; 1404 } 1405 1406 const std::string& CompositeSource::GetIdentifier() const 1407 { 1408 return m_details.Identifier; 1409 } 1410 1411 // The composite search needs to take several steps to get results, and due to the 1412 // potential for different information spread across multiple sources, base searches 1413 // need to be performed in both installed and available. 1414 // 1415 // If an installed source is present, then the searches should only return packages 1416 // that are installed. This means that the base searches against available sources 1417 // will only return results where a match is found in the installed source. 1418 SearchResult CompositeSource::Search(const SearchRequest& request) const 1419 { 1420 if (m_installedSource) 1421 { 1422 return SearchInstalled(request); 1423 } 1424 else 1425 { 1426 return SearchAvailable(request); 1427 } 1428 } 1429 1430 void* CompositeSource::CastTo(ISourceType type) 1431 { 1432 if (type == SourceType) 1433 { 1434 return this; 1435 } 1436 1437 return nullptr; 1438 } 1439 1440 void CompositeSource::AddAvailableSource(const Source& source) 1441 { 1442 m_availableSources.emplace_back(source); 1443 } 1444 1445 void CompositeSource::SetInstalledSource(Source source, CompositeSearchBehavior searchBehavior) 1446 { 1447 m_installedSource = std::move(source); 1448 m_searchBehavior = searchBehavior; 1449 } 1450 1451 // An installed search first finds all installed packages that match the request, then correlates with available sources. 1452 // Next the search is performed against the available sources and correlated with the installed source. A result will only 1453 // be added if there exists an installed package that was not found by the initial search. 1454 // This allows for search terms to find installed packages by their available metadata, as well as the local values. 1455 // 1456 // Search flow: 1457 // Installed :: Search incoming request 1458 // For each result 1459 // For each available source 1460 // Tracking :: Search system references 1461 // If tracking found 1462 // Available :: Search tracking ID 1463 // If no available, for each available source 1464 // Available :: Search system references 1465 // 1466 // For each available source 1467 // Tracking :: Search incoming request 1468 // For each result 1469 // Installed :: Search system references 1470 // If found 1471 // Available :: Search tracking ID 1472 // Available :: Search incoming request 1473 // For each result 1474 // Installed :: Search system references 1475 SearchResult CompositeSource::SearchInstalled(const SearchRequest& request) const 1476 { 1477 CompositeResult result; 1478 1479 // If the search behavior is for AllPackages or Installed then the result can contain packages that are 1480 // only in the Installed source, but do not have an AvailableVersion. 1481 if (m_searchBehavior == CompositeSearchBehavior::AllPackages || m_searchBehavior == CompositeSearchBehavior::Installed) 1482 { 1483 // Search installed source (allow exceptions out as we own the installed source) 1484 SearchResult installedResult = m_installedSource.Search(request); 1485 result.Truncated = installedResult.Truncated; 1486 1487 for (auto&& match : installedResult.Matches) 1488 { 1489 if (!match.Package) 1490 { 1491 // Ensure that the crash from installedVersion below is not from the actual package being null. 1492 AICLI_LOG(Repo, Warning, << "CompositeSource: The match of the package (matched on " << 1493 ToString(match.MatchCriteria.Field) << " => '" << match.MatchCriteria.Value << 1494 "') was null and is being dropped from the results."); 1495 continue; 1496 } 1497 1498 std::shared_ptr<CompositePackage> compositePackage = std::make_shared<CompositePackage>(match.Package); 1499 auto installedPackage = compositePackage->GetInstalled(); 1500 1501 if (!installedPackage) 1502 { 1503 // One would think that the installed package coming directly from our own installed source 1504 // would never be null, but it is sometimes. Rather than making users suffer through crashes 1505 // that break their entire experience, lets log a few things and then ignore this match. 1506 AICLI_LOG(Repo, Warning, << "CompositeSource: The installed version of the package '" << 1507 match.Package->GetProperty(PackageProperty::Id) << "' was null and is being dropped from the results."); 1508 continue; 1509 } 1510 1511 auto installedPackageData = result.GetSystemReferenceStrings(installedPackage.get()); 1512 1513 // Create a search request to run against all available sources 1514 if (!installedPackageData.SystemReferenceStrings.empty()) 1515 { 1516 SearchRequest systemReferenceSearch = installedPackageData.CreateInclusionsSearchRequest(SearchPurpose::CorrelationToAvailable); 1517 AICLI_LOG(Repo, Verbose, << "Finding available package from installed package using system reference search: " << systemReferenceSearch.ToString()); 1518 1519 // Search sources and add to result 1520 for (const auto& source : m_availableSources) 1521 { 1522 AICLI_LOG(Repo, Verbose, << " ... searching source: " << source.GetDetails().Name << " [" << source.GetIdentifier() << ']'); 1523 1524 // Find the tracking result with the latest timestamp. 1525 auto trackingCatalog = source.GetTrackingCatalog(); 1526 SearchResult trackingResult = trackingCatalog.Search(systemReferenceSearch); 1527 1528 std::shared_ptr<IPackage> trackingPackage; 1529 std::chrono::system_clock::time_point trackingPackageTime; 1530 bool trackingSet = false; 1531 1532 for (const auto& trackingMatch : trackingResult.Matches) 1533 { 1534 auto candidateTime = GetLatestTrackingWriteTime(OnlyAvailable(trackingMatch.Package)); 1535 1536 if (!trackingPackage || candidateTime > trackingPackageTime) 1537 { 1538 trackingPackage = OnlyAvailable(trackingMatch.Package); 1539 trackingPackageTime = candidateTime; 1540 } 1541 } 1542 1543 if (trackingPackage && trackingPackageTime > compositePackage->GetTrackingPackageWriteTime()) 1544 { 1545 AICLI_LOG(Repo, Verbose, << " ... setting latest tracking package to: " << trackingPackage->GetProperty(PackageProperty::Id)); 1546 compositePackage->SetTracking(source, trackingPackage, trackingPackageTime); 1547 trackingSet = true; 1548 } 1549 1550 // Attempt to correlate local packages against this source if supported. 1551 SearchResult availableResult; 1552 if (source.GetDetails().SupportInstalledSearchCorrelation) 1553 { 1554 availableResult = result.SearchAndHandleFailures(source, systemReferenceSearch); 1555 } 1556 1557 auto availablePackage = GetMatchingPackage(availableResult.Matches, 1558 [&]() { 1559 AICLI_LOG(Repo, Info, 1560 << "Found multiple matches for installed package [" << installedPackage->GetProperty(PackageProperty::Id) << 1561 "] in source [" << source.GetIdentifier() << "] when searching for [" << systemReferenceSearch.ToString() << "]"); 1562 }, [&] { 1563 AICLI_LOG(Repo, Warning, << " Appropriate available package could not be determined"); 1564 }); 1565 1566 if (trackingPackage) 1567 { 1568 auto trackingIdentifier = trackingPackage->GetProperty(PackageProperty::Id); 1569 1570 // We always want to take the available search result if it exists as the package may have been updated. 1571 if (availablePackage) 1572 { 1573 auto availableIdentifier = availablePackage->GetProperty(PackageProperty::Id); 1574 if (!Utility::ICUCaseInsensitiveEquals(availableIdentifier, trackingIdentifier)) 1575 { 1576 AICLI_LOG(Repo, Verbose, << " ... overriding tracking package (" << trackingIdentifier << ") with available package (" << availableIdentifier << ")"); 1577 } 1578 } 1579 else 1580 { 1581 AICLI_LOG(Repo, Verbose, << " ... using tracking package: " << trackingIdentifier); 1582 availablePackage = GetTrackedPackageFromAvailableSource(result, source, trackingIdentifier); 1583 } 1584 } 1585 1586 if (availablePackage) 1587 { 1588 AICLI_LOG(Repo, Verbose, << " ... adding available package: " << availablePackage->GetProperty(PackageProperty::Id)); 1589 compositePackage->AddAvailablePackage(availablePackage, trackingSet); 1590 } 1591 } 1592 } 1593 1594 // Move the installed result into the composite result 1595 result.Matches.emplace_back(std::move(compositePackage), std::move(match.MatchCriteria)); 1596 } 1597 1598 // Optimization for the "everything installed" case, no need to allow for reverse correlations 1599 if (request.IsForEverything() && m_searchBehavior == CompositeSearchBehavior::Installed) 1600 { 1601 return result.ConvertToSearchResult(); 1602 } 1603 } 1604 1605 // Search available sources 1606 for (const auto& source : m_availableSources) 1607 { 1608 auto trackingCatalog = source.GetTrackingCatalog(); 1609 1610 SearchResult availableResult = result.SearchAndHandleFailures(source, request); 1611 bool downloadManifests = source.QueryFeatureFlag(SourceFeatureFlag::ManifestMayContainAdditionalSystemReferenceStrings); 1612 1613 for (auto&& match : availableResult.Matches) 1614 { 1615 // Check for the package already in the result. 1616 // In cases that PackageData will be created, also download manifests for system reference strings 1617 // when search result is small (currently limiting to 1). 1618 auto packageData = result.CheckForExistingResultFromAvailablePackageMatch(match, downloadManifests && availableResult.Matches.size() == 1); 1619 1620 // If found existing package in the result, continue 1621 if (!packageData) 1622 { 1623 continue; 1624 } 1625 1626 // Use data from the tracking catalog as it can potentially get better correlations 1627 auto trackingPackage = packageData->AddSystemReferenceStringsFromTrackingPackage(trackingCatalog, match.Package->GetProperty(PackageProperty::Id), source.GetDetails().Name); 1628 1629 // If no package was found that was already in the results, do a correlation lookup with the installed 1630 // source to create a new composite package entry if we find any packages there. 1631 bool foundInstalledMatch = false; 1632 if (!packageData->SystemReferenceStrings.empty()) 1633 { 1634 // Create a search request to run against the installed source 1635 SearchRequest systemReferenceSearch = packageData->CreateInclusionsSearchRequest(SearchPurpose::CorrelationToInstalled); 1636 1637 AICLI_LOG(Repo, Verbose, << "Finding installed package from available package using system reference search: " << systemReferenceSearch.ToString()); 1638 // Correlate against installed (allow exceptions out as we own the installed source) 1639 SearchResult installedCrossRef = m_installedSource.Search(systemReferenceSearch); 1640 1641 for (const auto& installedMatch : installedCrossRef.Matches) 1642 { 1643 if (!IsStrongMatchField(installedMatch.MatchCriteria.Field)) 1644 { 1645 // For weak correlations, do an installed -> available check to ensure that there are no other strong correlations. 1646 SearchResult correlationConfirmation; 1647 if (source.GetDetails().SupportInstalledSearchCorrelation) 1648 { 1649 correlationConfirmation = result.SearchAndHandleFailures(source, result.GetSystemReferenceStrings(installedMatch.Package->GetInstalled().get()).CreateInclusionsSearchRequest(SearchPurpose::CorrelationToAvailable)); 1650 } 1651 1652 if (correlationConfirmation.Matches.empty()) 1653 { 1654 // We probably made the correlation due to tracking data, keep it. 1655 } 1656 else if (correlationConfirmation.Matches.size() > 1) 1657 { 1658 // There is contention for the correlation. 1659 AICLI_LOG(Repo, Verbose, << " ... installed package [" << installedMatch.Package->GetProperty(PackageProperty::Id) << 1660 "] had multiple correlations and is being ignored as a match for [" << match.Package->GetProperty(PackageProperty::Id) << "]"); 1661 continue; 1662 } 1663 else if (!OnlyAvailable(correlationConfirmation.Matches[0].Package)->IsSame(OnlyAvailable(match.Package).get())) 1664 { 1665 // The only correlation is not to the current package. 1666 AICLI_LOG(Repo, Verbose, << " ... installed package [" << installedMatch.Package->GetProperty(PackageProperty::Id) << 1667 "] was found through available package [" << match.Package->GetProperty(PackageProperty::Id) << "], but only correlated to [" << 1668 correlationConfirmation.Matches[0].Package->GetProperty(PackageProperty::Id) << "] and is being ignored"); 1669 continue; 1670 } 1671 } 1672 1673 // Now that we know we need to add this available package, determine how exactly 1674 std::shared_ptr<CompositePackage> resultPackage = result.FindInstalledPackage(installedMatch.Package->GetInstalled().get()); 1675 1676 if (resultPackage) 1677 { 1678 // Check for a package from the same source already present on the result package. 1679 bool foundSameSource = false; 1680 1681 for (const auto& availablePackage : resultPackage->GetAvailablePackages()) 1682 { 1683 if (availablePackage->GetSource() == source) 1684 { 1685 // TODO: May need to add more data so that we can choose the proper correlation, but it may also be very difficult to get through 1686 // the gauntlet of other checks and arrive in this situation. 1687 AICLI_LOG(Repo, Verbose, << " ... found [" << availablePackage->GetProperty(PackageProperty::Id) << 1688 "] already correlated to [" << installedMatch.Package->GetProperty(PackageProperty::Id) << "] from the same source [" << 1689 source.GetDetails().Name << "] as [" << match.Package->GetProperty(PackageProperty::Id) << "]; ignoring the second correlation"); 1690 foundSameSource = true; 1691 } 1692 } 1693 1694 if (foundSameSource) 1695 { 1696 continue; 1697 } 1698 } 1699 else 1700 { 1701 result.Matches.emplace_back(std::make_shared<CompositePackage>(installedMatch.Package), match.MatchCriteria); 1702 resultPackage = result.Matches.back().Package; 1703 } 1704 1705 bool setPrimary = false; 1706 if (trackingPackage) 1707 { 1708 auto trackingPackageTime = GetLatestTrackingWriteTime(trackingPackage); 1709 1710 if (trackingPackageTime > resultPackage->GetTrackingPackageWriteTime()) 1711 { 1712 resultPackage->SetTracking(source, std::move(trackingPackage), trackingPackageTime); 1713 setPrimary = true; 1714 } 1715 } 1716 1717 resultPackage->AddAvailablePackage(std::move(match.Package), setPrimary); 1718 1719 foundInstalledMatch = true; 1720 } 1721 } 1722 1723 // If there was no correlation for this package, add it without one. 1724 if ((m_searchBehavior == CompositeSearchBehavior::AllPackages || m_searchBehavior == CompositeSearchBehavior::AvailablePackages) && !foundInstalledMatch) 1725 { 1726 result.Matches.emplace_back(std::make_shared<CompositePackage>(std::shared_ptr<ICompositePackage>{}, std::move(match.Package)), match.MatchCriteria); 1727 } 1728 } 1729 } 1730 1731 SortResultMatches(result.Matches); 1732 1733 if (request.MaximumResults > 0 && result.Matches.size() > request.MaximumResults) 1734 { 1735 result.Truncated = true; 1736 result.Matches.erase(result.Matches.begin() + request.MaximumResults, result.Matches.end()); 1737 } 1738 1739 return result.ConvertToSearchResult(); 1740 } 1741 1742 // An available search goes through each source, searching individually and then sorting the full result set. 1743 SearchResult CompositeSource::SearchAvailable(const SearchRequest& request) const 1744 { 1745 SearchResult result; 1746 1747 // Search available sources 1748 for (const auto& source : m_availableSources) 1749 { 1750 SearchResult oneSourceResult; 1751 1752 try 1753 { 1754 oneSourceResult = source.Search(request); 1755 } 1756 catch (...) 1757 { 1758 LOG_CAUGHT_EXCEPTION(); 1759 AICLI_LOG(Repo, Warning, << "Failed to search source: " << source.GetDetails().Name); 1760 result.Failures.emplace_back(SearchResult::Failure{ source.GetDetails().Name, std::current_exception() }); 1761 } 1762 1763 // Move into the single result 1764 std::move(oneSourceResult.Matches.begin(), oneSourceResult.Matches.end(), std::back_inserter(result.Matches)); 1765 std::move(oneSourceResult.Failures.begin(), oneSourceResult.Failures.end(), std::back_inserter(result.Failures)); 1766 } 1767 1768 SortResultMatches(result.Matches); 1769 1770 if (request.MaximumResults > 0 && result.Matches.size() > request.MaximumResults) 1771 { 1772 result.Truncated = true; 1773 result.Matches.erase(result.Matches.begin() + request.MaximumResults, result.Matches.end()); 1774 } 1775 1776 return result; 1777 } 1778 }