Downloader.cpp (23504B)
1 // Copyright (c) Microsoft Corporation. 2 // Licensed under the MIT License. 3 #include "pch.h" 4 #include <wininet.h> 5 #include "Public/AppInstallerErrors.h" 6 #include "Public/AppInstallerRuntime.h" 7 #include "Public/AppInstallerDownloader.h" 8 #include "Public/AppInstallerSHA256.h" 9 #include "Public/AppInstallerStrings.h" 10 #include "Public/AppInstallerLogging.h" 11 #include "Public/AppInstallerTelemetry.h" 12 #include "Public/winget/UserSettings.h" 13 #include "Public/winget/NetworkSettings.h" 14 #include "Public/winget/Filesystem.h" 15 #include "DODownloader.h" 16 #include "HttpStream/HttpRandomAccessStream.h" 17 18 using namespace AppInstaller::Runtime; 19 using namespace AppInstaller::Settings; 20 using namespace AppInstaller::Filesystem; 21 using namespace AppInstaller::Utility::HttpStream; 22 using namespace winrt::Windows::Web::Http; 23 using namespace winrt::Windows::Web::Http::Headers; 24 using namespace winrt::Windows::Web::Http::Filters; 25 26 namespace AppInstaller::Utility 27 { 28 namespace 29 { 30 std::wstring GetHttpQueryString(const wil::unique_hinternet& urlFile, DWORD queryProperty) 31 { 32 std::wstring result = {}; 33 DWORD length = 0; 34 if (!HttpQueryInfoW(urlFile.get(), 35 queryProperty, 36 &result[0], 37 &length, 38 nullptr)) 39 { 40 auto lastError = GetLastError(); 41 if (lastError == ERROR_INSUFFICIENT_BUFFER) 42 { 43 // lpdwBufferLength contains the size, in bytes, of a buffer large enough to receive the requested information 44 // without the nul char. not the exact buffer size. 45 auto size = static_cast<size_t>(length) / sizeof(wchar_t); 46 result.resize(size + 1); 47 if (HttpQueryInfoW(urlFile.get(), 48 queryProperty, 49 &result[0], 50 &length, 51 nullptr)) 52 { 53 // because the buffer can be bigger remove possible null chars 54 result.erase(result.find(L'\0')); 55 } 56 else 57 { 58 AICLI_LOG(Core, Error, << "Error retrieving header value [" << queryProperty << "]: " << GetLastError()); 59 result.clear(); 60 } 61 } 62 else 63 { 64 AICLI_LOG(Core, Error, << "Error retrieving header [" << queryProperty << "]: " << GetLastError()); 65 } 66 } 67 68 return result; 69 } 70 71 // Gets the retry after value in terms of a delay in seconds 72 std::chrono::seconds GetRetryAfter(const HttpDateOrDeltaHeaderValue& retryAfter) 73 { 74 if (retryAfter) 75 { 76 auto delta = retryAfter.Delta(); 77 if (delta) 78 { 79 return std::chrono::duration_cast<std::chrono::seconds>(delta.GetTimeSpan()); 80 } 81 82 auto dateTimeRef = retryAfter.Date(); 83 if (dateTimeRef) 84 { 85 auto dateTime = dateTimeRef.GetDateTime(); 86 auto now = winrt::clock::now(); 87 88 if (dateTime > now) 89 { 90 return std::chrono::duration_cast<std::chrono::seconds>(dateTime - now); 91 } 92 } 93 } 94 95 return 0s; 96 } 97 98 std::chrono::seconds GetRetryAfter(const wil::unique_hinternet& urlFile) 99 { 100 std::wstring retryAfter = GetHttpQueryString(urlFile, HTTP_QUERY_RETRY_AFTER); 101 return retryAfter.empty() ? 0s : AppInstaller::Utility::GetRetryAfter(retryAfter); 102 } 103 } 104 105 #ifndef AICLI_DISABLE_TEST_HOOKS 106 namespace TestHooks 107 { 108 static std::function<DownloadResult( 109 const std::string& url, 110 const std::filesystem::path& dest, 111 DownloadType type, 112 IProgressCallback& progress, 113 std::optional<DownloadInfo> info)>* s_Download_Function_Override = nullptr; 114 115 void SetDownloadResult_Function_Override(std::function<DownloadResult( 116 const std::string& url, 117 const std::filesystem::path& dest, 118 DownloadType type, 119 IProgressCallback& progress, 120 std::optional<DownloadInfo> info)>* value) 121 { 122 s_Download_Function_Override = value; 123 } 124 } 125 #endif 126 127 DownloadResult WinINetDownloadToStream( 128 const std::string& url, 129 std::ostream& dest, 130 IProgressCallback& progress, 131 std::optional<DownloadInfo> info) 132 { 133 // For AICLI_LOG usages with string literals. 134 #pragma warning(push) 135 #pragma warning(disable:26449) 136 137 AICLI_LOG(Core, Info, << "WinINet downloading from url: " << url); 138 139 auto agentWide = Utility::ConvertToUTF16(Runtime::GetDefaultUserAgent().get()); 140 wil::unique_hinternet session; 141 142 const auto& proxyUri = Network().GetProxyUri(); 143 if (proxyUri) 144 { 145 AICLI_LOG(Core, Info, << "Using proxy " << proxyUri.value()); 146 session.reset(InternetOpen( 147 agentWide.c_str(), 148 INTERNET_OPEN_TYPE_PROXY, 149 Utility::ConvertToUTF16(proxyUri.value()).c_str(), 150 NULL, 151 0)); 152 } 153 else 154 { 155 session.reset(InternetOpen( 156 agentWide.c_str(), 157 INTERNET_OPEN_TYPE_PRECONFIG, 158 NULL, 159 NULL, 160 0)); 161 } 162 163 THROW_LAST_ERROR_IF_NULL_MSG(session, "InternetOpen() failed."); 164 165 std::string customHeaders; 166 if (info && info->RequestHeaders.size() > 0) 167 { 168 for (const auto& header : info->RequestHeaders) 169 { 170 customHeaders += header.Name + ": " + header.Value + "\r\n"; 171 } 172 } 173 std::wstring customHeadersWide = Utility::ConvertToUTF16(customHeaders); 174 175 auto urlWide = Utility::ConvertToUTF16(url); 176 wil::unique_hinternet urlFile(InternetOpenUrl( 177 session.get(), 178 urlWide.c_str(), 179 customHeadersWide.empty() ? NULL : customHeadersWide.c_str(), 180 customHeadersWide.empty() ? 0 : (DWORD)(customHeadersWide.size()), 181 INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTPS, // This allows http->https redirection 182 0)); 183 THROW_LAST_ERROR_IF_NULL_MSG(urlFile, "InternetOpenUrl() failed."); 184 185 // Check http return status 186 DWORD requestStatus = 0; 187 DWORD cbRequestStatus = sizeof(requestStatus); 188 189 THROW_LAST_ERROR_IF_MSG(!HttpQueryInfoW(urlFile.get(), 190 HTTP_QUERY_STATUS_CODE | HTTP_QUERY_FLAG_NUMBER, 191 &requestStatus, 192 &cbRequestStatus, 193 nullptr), "Query download request status failed."); 194 195 constexpr DWORD TooManyRequest = 429; 196 197 switch (requestStatus) 198 { 199 case HTTP_STATUS_OK: 200 // All good 201 break; 202 case TooManyRequest: 203 case HTTP_STATUS_SERVICE_UNAVAIL: 204 { 205 THROW_EXCEPTION(ServiceUnavailableException(GetRetryAfter(urlFile))); 206 } 207 default: 208 AICLI_LOG(Core, Error, << "Download request failed. Returned status: " << requestStatus); 209 THROW_HR_MSG(MAKE_HRESULT(SEVERITY_ERROR, FACILITY_HTTP, requestStatus), "Download request status is not success."); 210 } 211 212 AICLI_LOG(Core, Verbose, << "Download request status success."); 213 214 // Get content length. Don't fail the download if failed. 215 LONGLONG contentLength = 0; 216 DWORD cbContentLength = sizeof(contentLength); 217 218 HttpQueryInfoW( 219 urlFile.get(), 220 HTTP_QUERY_CONTENT_LENGTH | HTTP_QUERY_FLAG_NUMBER64, 221 &contentLength, 222 &cbContentLength, 223 nullptr); 224 AICLI_LOG(Core, Verbose, << "Download size: " << contentLength); 225 226 std::string contentType = Utility::ConvertToUTF8(GetHttpQueryString(urlFile, HTTP_QUERY_CONTENT_TYPE)); 227 AICLI_LOG(Core, Verbose, << "Content Type: " << contentType); 228 229 // Setup hash engine 230 SHA256 hashEngine; 231 232 const int bufferSize = 1024 * 1024; // 1MB 233 auto buffer = std::make_unique<BYTE[]>(bufferSize); 234 235 BOOL readSuccess = true; 236 DWORD bytesRead = 0; 237 LONGLONG bytesDownloaded = 0; 238 239 do 240 { 241 if (progress.IsCancelledBy(CancelReason::Any)) 242 { 243 AICLI_LOG(Core, Info, << "Download cancelled."); 244 return {}; 245 } 246 247 readSuccess = InternetReadFile(urlFile.get(), buffer.get(), bufferSize, &bytesRead); 248 249 THROW_LAST_ERROR_IF_MSG(!readSuccess, "InternetReadFile() failed."); 250 251 hashEngine.Add(buffer.get(), bytesRead); 252 253 dest.write((char*)buffer.get(), bytesRead); 254 255 bytesDownloaded += bytesRead; 256 257 if (bytesRead != 0) 258 { 259 progress.OnProgress(bytesDownloaded, contentLength, ProgressType::Bytes); 260 } 261 262 } while (bytesRead != 0); 263 264 dest.flush(); 265 266 // Check download size matches if content length is provided in response header 267 if (contentLength > 0) 268 { 269 THROW_HR_IF(APPINSTALLER_CLI_ERROR_DOWNLOAD_SIZE_MISMATCH, bytesDownloaded != contentLength); 270 } 271 272 DownloadResult result; 273 result.SizeInBytes = static_cast<uint64_t>(bytesDownloaded); 274 result.ContentType = std::move(contentType); 275 result.Sha256Hash = hashEngine.Get(); 276 AICLI_LOG(Core, Info, << "Download hash: " << SHA256::ConvertToString(result.Sha256Hash)); 277 278 AICLI_LOG(Core, Info, << "Download completed."); 279 280 #pragma warning(pop) 281 282 return result; 283 } 284 285 std::map<std::string, std::string> GetHeaders(std::string_view url) 286 { 287 // TODO: Use proxy info. HttpClient does not support using a custom proxy, only using the system-wide one. 288 AICLI_LOG(Core, Verbose, << "Retrieving headers from url: " << url); 289 290 HttpBaseProtocolFilter filter; 291 filter.CacheControl().ReadBehavior(HttpCacheReadBehavior::MostRecent); 292 293 HttpClient client(filter); 294 client.DefaultRequestHeaders().Connection().Clear(); 295 client.DefaultRequestHeaders().Append(L"Connection", L"close"); 296 client.DefaultRequestHeaders().UserAgent().ParseAdd(Utility::ConvertToUTF16(Runtime::GetDefaultUserAgent().get())); 297 298 winrt::Windows::Foundation::Uri uri{ Utility::ConvertToUTF16(url) }; 299 HttpRequestMessage request(HttpMethod::Head(), uri); 300 301 HttpResponseMessage response = client.SendRequestAsync(request, HttpCompletionOption::ResponseHeadersRead).get(); 302 303 switch (response.StatusCode()) 304 { 305 case HttpStatusCode::Ok: 306 // All good 307 break; 308 case HttpStatusCode::TooManyRequests: 309 case HttpStatusCode::ServiceUnavailable: 310 { 311 THROW_EXCEPTION(ServiceUnavailableException(GetRetryAfter(response.Headers().RetryAfter()))); 312 } 313 default: 314 THROW_HR(MAKE_HRESULT(SEVERITY_ERROR, FACILITY_HTTP, response.StatusCode())); 315 } 316 317 std::map<std::string, std::string> result; 318 319 for (const auto& header : response.Headers()) 320 { 321 result.emplace(Utility::FoldCase(static_cast<std::string_view>(Utility::ConvertToUTF8(header.Key()))), Utility::ConvertToUTF8(header.Value())); 322 } 323 324 return result; 325 } 326 327 DownloadResult DownloadToStream( 328 const std::string& url, 329 std::ostream& dest, 330 DownloadType, 331 IProgressCallback& progress, 332 std::optional<DownloadInfo> info) 333 { 334 THROW_HR_IF(E_INVALIDARG, url.empty()); 335 return WinINetDownloadToStream(url, dest, progress, info); 336 } 337 338 DownloadResult Download( 339 const std::string& url, 340 const std::filesystem::path& dest, 341 DownloadType type, 342 IProgressCallback& progress, 343 std::optional<DownloadInfo> info) 344 { 345 #ifndef AICLI_DISABLE_TEST_HOOKS 346 if (TestHooks::s_Download_Function_Override) 347 { 348 return (*TestHooks::s_Download_Function_Override)(url, dest, type, progress, info); 349 } 350 #endif 351 352 THROW_HR_IF(E_INVALIDARG, url.empty()); 353 THROW_HR_IF(E_INVALIDARG, dest.empty()); 354 355 AICLI_LOG(Core, Info, << "Downloading to path: " << dest); 356 357 std::filesystem::create_directories(dest.parent_path()); 358 359 // Only Installers should be downloaded with DO currently, as: 360 // - Index :: Constantly changing blob at same location is not what DO is for 361 // - Manifest / InstallerMetadataCollectionInput :: DO overhead is not needed for small files 362 // - WinGetUtil :: Intentionally not using DO at this time 363 if (type == DownloadType::Installer) 364 { 365 if (Network().GetInstallerDownloader() == InstallerDownloader::DeliveryOptimization) 366 { 367 try 368 { 369 auto result = DODownload(url, dest, progress, info); 370 // Since we cannot pre-apply to the file with DO, post-apply the MotW to the file. 371 // Only do so if the file exists, because cancellation will not throw here. 372 if (std::filesystem::exists(dest)) 373 { 374 ApplyMotwIfApplicable(dest, URLZONE_INTERNET); 375 } 376 return result; 377 } 378 catch (const wil::ResultException& re) 379 { 380 // Fall back to WinINet below unless the specific error is not one that should be ignored. 381 // We need to be careful not to bypass metered networks or other reasons that might 382 // intentionally cause the download to be blocked. 383 HRESULT hr = re.GetErrorCode(); 384 if (IsDOErrorFatal(hr)) 385 { 386 throw; 387 } 388 else 389 { 390 // Send telemetry so that we can understand the reasons for DO failing 391 Logging::Telemetry().LogNonFatalDOError(url, hr); 392 } 393 } 394 395 // If we reach this point, we are intending to fall through to WinINet. 396 // Remove any file that may have been placed in the target location. 397 if (std::filesystem::exists(dest)) 398 { 399 std::filesystem::remove(dest); 400 } 401 } 402 } 403 404 std::ofstream emptyDestFile(dest); 405 emptyDestFile.close(); 406 ApplyMotwIfApplicable(dest, URLZONE_INTERNET); 407 408 // Use std::ofstream::app to append to previous empty file so that it will not 409 // create a new file and clear motw. 410 std::ofstream outfile(dest, std::ofstream::binary | std::ofstream::app); 411 return WinINetDownloadToStream(url, outfile, progress, info); 412 } 413 414 using namespace std::string_view_literals; 415 constexpr std::string_view s_http_start = "http://"sv; 416 constexpr std::string_view s_https_start = "https://"sv; 417 418 bool IsUrlRemote(std::string_view url) 419 { 420 // Very simple choice right now: "does it start with http:// or https://"? 421 if (CaseInsensitiveStartsWith(url, s_http_start) || 422 CaseInsensitiveStartsWith(url, s_https_start)) 423 { 424 return true; 425 } 426 427 return false; 428 } 429 430 bool IsUrlSecure(std::string_view url) 431 { 432 // Very simple choice right now: "does it start with https://"? 433 if (CaseInsensitiveStartsWith(url, s_https_start)) 434 { 435 return true; 436 } 437 438 return false; 439 } 440 441 static inline bool FileSupportsMotw(const std::filesystem::path& path) 442 { 443 return SupportsNamedStreams(path); 444 } 445 446 void ApplyMotwIfApplicable(const std::filesystem::path& filePath, URLZONE zone) 447 { 448 AICLI_LOG(Core, Info, << "Started applying motw to " << filePath << " with zone: " << zone); 449 450 if (!FileSupportsMotw(filePath)) 451 { 452 AICLI_LOG(Core, Info, << "File system does not support ADS. Skipped applying motw"); 453 return; 454 } 455 456 Microsoft::WRL::ComPtr<IZoneIdentifier> zoneIdentifier; 457 THROW_IF_FAILED(CoCreateInstance(CLSID_PersistentZoneIdentifier, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&zoneIdentifier))); 458 THROW_IF_FAILED(zoneIdentifier->SetId(zone)); 459 460 Microsoft::WRL::ComPtr<IPersistFile> persistFile; 461 THROW_IF_FAILED(zoneIdentifier.As(&persistFile)); 462 THROW_IF_FAILED(persistFile->Save(filePath.c_str(), TRUE)); 463 464 AICLI_LOG(Core, Info, << "Finished applying motw"); 465 } 466 467 void RemoveMotwIfApplicable(const std::filesystem::path& filePath) 468 { 469 AICLI_LOG(Core, Info, << "Started removing motw to " << filePath); 470 471 if (!FileSupportsMotw(filePath)) 472 { 473 AICLI_LOG(Core, Info, << "File system does not support ADS. Skipped removing motw"); 474 return; 475 } 476 477 Microsoft::WRL::ComPtr<IZoneIdentifier> zoneIdentifier; 478 THROW_IF_FAILED(CoCreateInstance(CLSID_PersistentZoneIdentifier, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&zoneIdentifier))); 479 480 Microsoft::WRL::ComPtr<IPersistFile> persistFile; 481 THROW_IF_FAILED(zoneIdentifier.As(&persistFile)); 482 483 auto hr = persistFile->Load(filePath.c_str(), STGM_READ); 484 if (hr == HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND)) 485 { 486 // IPersistFile::Load returns same error for "file not found" and "motw not found". 487 // Check if the file exists to be sure we are on the "motw not found" case. 488 THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND), !std::filesystem::exists(filePath)); 489 490 AICLI_LOG(Core, Info, << "File does not contain motw. Skipped removing motw"); 491 return; 492 } 493 494 THROW_IF_FAILED(zoneIdentifier->Remove()); 495 THROW_IF_FAILED(persistFile->Save(NULL, TRUE)); 496 497 AICLI_LOG(Core, Info, << "Finished removing motw"); 498 } 499 500 HRESULT ApplyMotwUsingIAttachmentExecuteIfApplicable(const std::filesystem::path& filePath, const std::string& source, URLZONE zoneIfScanFailure) 501 { 502 AICLI_LOG(Core, Info, << "Started applying motw using IAttachmentExecute to " << filePath); 503 504 if (!FileSupportsMotw(filePath)) 505 { 506 AICLI_LOG(Core, Info, << "File system does not support ADS. Skipped applying motw"); 507 return S_OK; 508 } 509 510 // Attachment execution service needs STA to succeed, so we'll create a new thread and CoInitialize with STA. 511 HRESULT aesSaveResult = S_OK; 512 auto updateMotw = [&]() -> HRESULT 513 { 514 Microsoft::WRL::ComPtr<IAttachmentExecute> attachmentExecute; 515 RETURN_IF_FAILED(CoCreateInstance(CLSID_AttachmentServices, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&attachmentExecute))); 516 RETURN_IF_FAILED(attachmentExecute->SetLocalPath(filePath.c_str())); 517 RETURN_IF_FAILED(attachmentExecute->SetSource(Utility::ConvertToUTF16(source).c_str())); 518 519 // IAttachmentExecute::Save() expects the local file to be clean(i.e. it won't clear existing motw if it thinks the source url is trusted) 520 RemoveMotwIfApplicable(filePath); 521 522 aesSaveResult = attachmentExecute->Save(); 523 524 // Reapply desired zone upon scan failure. 525 // Not using SUCCEEDED(hr) to check since there are cases file is missing after a successful scan 526 if (aesSaveResult != S_OK && std::filesystem::exists(filePath)) 527 { 528 ApplyMotwIfApplicable(filePath, zoneIfScanFailure); 529 } 530 531 RETURN_IF_FAILED(aesSaveResult); 532 return S_OK; 533 }; 534 535 HRESULT hr = S_OK; 536 537 std::thread aesThread([&]() 538 { 539 hr = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED); 540 if (FAILED(hr)) 541 { 542 return; 543 } 544 545 hr = updateMotw(); 546 CoUninitialize(); 547 }); 548 549 aesThread.join(); 550 551 AICLI_LOG(Core, Info, << "Finished applying motw using IAttachmentExecute. Result: " << hr << " IAttachmentExecute::Save() result: " << aesSaveResult); 552 553 return aesSaveResult; 554 } 555 556 Microsoft::WRL::ComPtr<IStream> GetReadOnlyStreamFromURI(std::string_view uriStr) 557 { 558 Microsoft::WRL::ComPtr<IStream> inputStream; 559 if (Utility::IsUrlRemote(uriStr)) 560 { 561 // Get an IStream from the input uri and try to create package or bundler reader. 562 winrt::Windows::Foundation::Uri uri(Utility::ConvertToUTF16(uriStr)); 563 564 winrt::com_ptr<HttpRandomAccessStream> httpRandomAccessStream = winrt::make_self<HttpRandomAccessStream>(); 565 566 try 567 { 568 auto randomAccessStream = httpRandomAccessStream->InitializeAsync(uri).get(); 569 570 ::IUnknown* rasAsIUnknown = (::IUnknown*)winrt::get_abi(randomAccessStream); 571 THROW_IF_FAILED(CreateStreamOverRandomAccessStream( 572 rasAsIUnknown, 573 IID_PPV_ARGS(inputStream.ReleaseAndGetAddressOf()))); 574 } 575 catch (const winrt::hresult_error& hre) 576 { 577 if (hre.code() == APPINSTALLER_CLI_ERROR_SERVICE_UNAVAILABLE) 578 { 579 THROW_EXCEPTION(AppInstaller::Utility::ServiceUnavailableException(httpRandomAccessStream->RetryAfter())); 580 } 581 582 throw; 583 } 584 } 585 else 586 { 587 std::filesystem::path path(Utility::ConvertToUTF16(uriStr)); 588 THROW_IF_FAILED(SHCreateStreamOnFileEx(path.c_str(), 589 STGM_READ | STGM_SHARE_DENY_WRITE | STGM_FAILIFTHERE, 0, FALSE, nullptr, &inputStream)); 590 } 591 592 return inputStream; 593 } 594 595 std::chrono::seconds GetRetryAfter(const std::wstring& retryAfter) 596 { 597 try 598 { 599 winrt::hstring hstringValue{ retryAfter }; 600 HttpDateOrDeltaHeaderValue headerValue = nullptr; 601 HttpDateOrDeltaHeaderValue::TryParse(hstringValue, headerValue); 602 return GetRetryAfter(headerValue); 603 } 604 catch (...) 605 { 606 AICLI_LOG(Core, Error, << "Retry-After value not supported: " << Utility::ConvertToUTF8(retryAfter)); 607 } 608 609 return 0s; 610 } 611 612 std::chrono::seconds GetRetryAfter(const HttpResponseMessage& response) 613 { 614 return GetRetryAfter(response.Headers().RetryAfter()); 615 } 616 }