CurlConnectionTests.cpp (23741B)
1 // Copyright (c) Microsoft Corporation. 2 // Licensed under the MIT License. 3 4 #include "../../mock/MockWebServer.h" 5 #include "../../mock/ProxyServer.h" 6 #include "../../util/SFSExceptionMatcher.h" 7 #include "../../util/TestHelper.h" 8 #include "ReportingHandler.h" 9 #include "SFSUrlBuilder.h" 10 #include "TestOverride.h" 11 #include "connection/CurlConnection.h" 12 #include "connection/CurlConnectionManager.h" 13 #include "connection/HttpHeader.h" 14 15 #include <catch2/catch_test_macros.hpp> 16 #include <catch2/matchers/catch_matchers_string.hpp> 17 #include <curl/curl.h> 18 #include <nlohmann/json.hpp> 19 20 #include <chrono> 21 #include <sstream> 22 23 #define TEST(...) TEST_CASE("[Functional][CurlConnectionTests] " __VA_ARGS__) 24 25 using namespace SFS; 26 using namespace SFS::details; 27 using namespace SFS::test; 28 using namespace std::chrono; 29 using json = nlohmann::json; 30 31 const std::string c_instanceId = "default"; 32 const std::string c_namespace = "default"; 33 const std::string c_productName = "testProduct"; 34 const std::string c_version = "0.0.1"; 35 const std::string c_nextVersion = "0.0.2"; 36 37 namespace 38 { 39 class CurlConnectionTimeout : public CurlConnection 40 { 41 public: 42 CurlConnectionTimeout(const ConnectionConfig& config, const ReportingHandler& handler) 43 : CurlConnection(config, handler) 44 { 45 } 46 47 std::string Get(const std::string& url) override 48 { 49 // Timeout within 1ms 50 curl_easy_setopt(m_handle, CURLOPT_TIMEOUT_MS, 1L); 51 return CurlConnection::Get(url); 52 } 53 54 std::string Post(const std::string& url, const std::string& data) override 55 { 56 // Timeout within 1ms 57 curl_easy_setopt(m_handle, CURLOPT_TIMEOUT_MS, 1L); 58 return CurlConnection::Post(url, data); 59 } 60 }; 61 62 class CurlConnectionTimeoutManager : public CurlConnectionManager 63 { 64 public: 65 CurlConnectionTimeoutManager(const ReportingHandler& handler) : CurlConnectionManager(handler) 66 { 67 } 68 69 std::unique_ptr<Connection> MakeConnection(const ConnectionConfig& config) override 70 { 71 return std::make_unique<CurlConnectionTimeout>(config, m_handler); 72 } 73 }; 74 75 std::string TimestampToHttpDateString(std::chrono::time_point<std::chrono::system_clock> time) 76 { 77 auto timer = system_clock::to_time_t(time); 78 79 std::stringstream timeStream; 80 struct tm gmTime; 81 #ifdef _WIN32 82 gmtime_s(&gmTime, &timer); // gmtime_s is the safe version of gmtime, not available on Linux 83 #else 84 gmTime = (*std::gmtime(&timer)); 85 #endif 86 timeStream << std::put_time(&gmTime, "%a, %d %b %Y %X GMT"); // day, dd Mmm yyyy HH:MM:SS 87 88 return timeStream.str(); 89 } 90 } // namespace 91 92 TEST("Testing CurlConnection()") 93 { 94 test::MockWebServer server; 95 ReportingHandler handler; 96 handler.SetLoggingCallback(LogCallbackToTest); 97 CurlConnectionManager connectionManager(handler); 98 auto connection = connectionManager.MakeConnection({}); 99 SFSUrlBuilder urlBuilder(SFSCustomUrl(server.GetBaseUrl()), c_instanceId, c_namespace, handler); 100 server.RegisterExpectedRequestHeader(HttpHeader::UserAgent, GetUserAgentValue()); 101 102 SECTION("Testing CurlConnection::Get()") 103 { 104 const std::string url = urlBuilder.GetSpecificVersionUrl(c_productName, c_version); 105 106 json expectedResponse; 107 expectedResponse["ContentId"] = {{"Namespace", "default"}, {"Name", c_productName}, {"Version", c_version}}; 108 109 SECTION("Direct connection") 110 { 111 // Before registering the product, the URL returns 404 Not Found 112 REQUIRE_THROWS_CODE(connection->Get(url), HttpNotFound); 113 114 // Register the product 115 server.RegisterProduct(c_productName, c_version); 116 117 // After registering the product, the URL returns 200 OK 118 std::string out; 119 REQUIRE_NOTHROW(out = connection->Get(url)); 120 121 REQUIRE(json::parse(out) == expectedResponse); 122 } 123 124 SECTION("With proxy") 125 { 126 test::ProxyServer proxy; 127 128 ConnectionConfig config; 129 config.proxy = proxy.GetBaseUrl(); 130 connection = connectionManager.MakeConnection(config); 131 132 // Before registering the product, the URL returns 404 Not Found 133 REQUIRE_THROWS_CODE(connection->Get(url), HttpNotFound); 134 135 // Register the product 136 server.RegisterProduct(c_productName, c_version); 137 138 // After registering the product, the URL returns 200 OK 139 std::string out; 140 REQUIRE_NOTHROW(out = connection->Get(url)); 141 142 REQUIRE(json::parse(out) == expectedResponse); 143 144 REQUIRE(proxy.Stop() == Result::Success); 145 } 146 } 147 148 SECTION("Testing CurlConnection::Post()") 149 { 150 server.RegisterExpectedRequestHeader(HttpHeader::ContentType, "application/json"); 151 152 SECTION("With GetLatestVersionBatch mock") 153 { 154 const std::string url = urlBuilder.GetLatestVersionBatchUrl(); 155 156 // Missing proper body returns HttpBadRequest 157 REQUIRE_THROWS_CODE(connection->Post(url), HttpBadRequest); 158 159 // Before registering the product, the URL returns 404 Not Found 160 const json body = {{{"TargetingAttributes", {}}, {"Product", c_productName}}}; 161 REQUIRE_THROWS_CODE(connection->Post(url, body.dump()), HttpNotFound); 162 163 // Register the product 164 server.RegisterProduct(c_productName, c_version); 165 166 // After registering the product, the URL returns 200 OK 167 std::string out; 168 REQUIRE_NOTHROW(out = connection->Post(url, body.dump())); 169 170 json expectedResponse = json::array(); 171 expectedResponse.push_back( 172 {{"ContentId", {{"Namespace", "default"}, {"Name", c_productName}, {"Version", c_version}}}}); 173 REQUIRE(json::parse(out) == expectedResponse); 174 175 // Returns the next version now 176 server.RegisterProduct(c_productName, c_nextVersion); 177 REQUIRE_NOTHROW(out = connection->Post(url, body.dump())); 178 179 expectedResponse = json::array(); 180 expectedResponse.push_back( 181 {{"ContentId", {{"Namespace", "default"}, {"Name", c_productName}, {"Version", c_nextVersion}}}}); 182 REQUIRE(json::parse(out) == expectedResponse); 183 184 SECTION("Testing with proxy") 185 { 186 test::ProxyServer proxy; 187 188 ConnectionConfig config; 189 config.proxy = proxy.GetBaseUrl(); 190 connection = connectionManager.MakeConnection(config); 191 192 REQUIRE_NOTHROW(out = connection->Post(url, body.dump())); 193 194 REQUIRE(json::parse(out) == expectedResponse); 195 196 REQUIRE(proxy.Stop() == Result::Success); 197 } 198 } 199 200 SECTION("With GetDownloadInfo mock") 201 { 202 const std::string url = urlBuilder.GetDownloadInfoUrl(c_productName, c_version); 203 204 // Before registering the product, the URL returns 404 Not Found 205 REQUIRE_THROWS_CODE(connection->Post(url), HttpNotFound); 206 207 // Register the product 208 server.RegisterProduct(c_productName, c_version); 209 210 // After registering the product, the URL returns 200 OK 211 std::string out; 212 REQUIRE_NOTHROW(out = connection->Post(url)); 213 214 json expectedResponse = json::array(); 215 expectedResponse.push_back({{"Url", "http://localhost/1.json"}, 216 {"FileId", c_productName + ".json"}, 217 {"SizeInBytes", 100}, 218 {"Hashes", {{"Sha1", "123"}, {"Sha256", "456"}}}}); 219 expectedResponse[0]["DeliveryOptimization"] = {{"CatalogId", "789"}}; 220 expectedResponse[0]["DeliveryOptimization"]["Properties"] = { 221 {"IntegrityCheckInfo", {{"PiecesHashFileUrl", "http://localhost/1.json"}, {"HashOfHashes", "abc"}}}}; 222 223 expectedResponse.push_back({{"Url", "http://localhost/2.bin"}, 224 {"FileId", c_productName + ".bin"}, 225 {"SizeInBytes", 200}, 226 {"Hashes", {{"Sha1", "421"}, {"Sha256", "132"}}}}); 227 expectedResponse[1]["DeliveryOptimization"] = {{"CatalogId", "14"}}; 228 expectedResponse[1]["DeliveryOptimization"]["Properties"] = { 229 {"IntegrityCheckInfo", {{"PiecesHashFileUrl", "http://localhost/2.bin"}, {"HashOfHashes", "abcd"}}}}; 230 231 REQUIRE(json::parse(out) == expectedResponse); 232 233 SECTION("Testing with proxy") 234 { 235 test::ProxyServer proxy; 236 237 ConnectionConfig config; 238 config.proxy = proxy.GetBaseUrl(); 239 connection = connectionManager.MakeConnection(config); 240 241 REQUIRE_NOTHROW(out = connection->Post(url)); 242 243 REQUIRE(json::parse(out) == expectedResponse); 244 245 REQUIRE(proxy.Stop() == Result::Success); 246 } 247 } 248 } 249 250 SECTION("Testing with a malformed url") 251 { 252 std::string url = server.GetBaseUrl(); 253 REQUIRE_THROWS_CODE(connection->Get(url), HttpNotFound); 254 REQUIRE_THROWS_CODE(connection->Get(url + "/names"), HttpNotFound); 255 REQUIRE_THROWS_CODE(connection->Post(url + "/names", "{}"), HttpNotFound); 256 257 url = server.GetBaseUrl() + "/api/v1/contents/" + c_instanceId + "/namespaces/" + c_namespace + "/names/" + 258 c_productName + "/versYions/" + c_version + "/files?action=GenerateDownloadInfo"; 259 260 std::string out; 261 REQUIRE_THROWS_CODE(connection->Get(url), HttpNotFound); 262 REQUIRE_THROWS_CODE(connection->Post(url, std::string()), HttpNotFound); 263 REQUIRE_THROWS_CODE(out = connection->Post(url, {}), HttpNotFound); 264 CHECK(out.empty()); 265 } 266 267 REQUIRE(server.Stop() == Result::Success); 268 } 269 270 TEST("Testing CurlConnection when the server is not reachable") 271 { 272 // Using a custom override class just to time out faster on an invalid URL 273 ReportingHandler handler; 274 CurlConnectionTimeoutManager connectionManager(handler); 275 handler.SetLoggingCallback(LogCallbackToTest); 276 auto connection = connectionManager.MakeConnection({}); 277 278 // Using a non-routable IP address to ensure the server is not reachable 279 // https://www.rfc-editor.org/rfc/rfc5737#section-3: The blocks 192.0.2.0/24 (...) are provided for use in 280 // documentation. 281 std::string url = "192.0.2.0"; 282 std::string out; 283 REQUIRE_THROWS_CODE(connection->Get(url), HttpTimeout); 284 REQUIRE_THROWS_CODE(connection->Post(url + "/names", "{}"), HttpTimeout); 285 286 url = "192.0.2.0/files?action=GenerateDownloadInfo"; 287 288 REQUIRE_THROWS_CODE_MSG_MATCHES(connection->Get(url), 289 HttpTimeout, 290 Catch::Matchers::ContainsSubstring("timed out after")); 291 REQUIRE_THROWS_CODE_MSG_MATCHES(connection->Post(url, {}), 292 HttpTimeout, 293 Catch::Matchers::ContainsSubstring("timed out after")); 294 } 295 296 TEST("Testing CurlConnection works from a second ConnectionManager") 297 { 298 ReportingHandler handler; 299 300 // Create a first connection manager and see it leave scope - curl initializes and uninitializes 301 { 302 CurlConnectionManager connectionManager(handler); 303 REQUIRE(connectionManager.MakeConnection({}) != nullptr); 304 } 305 306 test::MockWebServer server; 307 CurlConnectionManager connectionManager(handler); 308 handler.SetLoggingCallback(LogCallbackToTest); 309 auto connection = connectionManager.MakeConnection({}); 310 SFSUrlBuilder urlBuilder(SFSCustomUrl(server.GetBaseUrl()), c_instanceId, c_namespace, handler); 311 312 const std::string url = urlBuilder.GetSpecificVersionUrl(c_productName, c_version); 313 314 // Register the product 315 server.RegisterProduct(c_productName, c_version); 316 317 // After registering the product, the URL returns 200 OK 318 REQUIRE_NOTHROW(connection->Get(url)); 319 320 SECTION("A second connection also works") 321 { 322 auto connection2 = connectionManager.MakeConnection({}); 323 REQUIRE_NOTHROW(connection2->Get(url)); 324 } 325 } 326 327 TEST("Testing a url that's too big throws 414") 328 { 329 ReportingHandler handler; 330 handler.SetLoggingCallback(LogCallbackToTest); 331 332 test::MockWebServer server; 333 CurlConnectionManager connectionManager(handler); 334 auto connection = connectionManager.MakeConnection({}); 335 SFSUrlBuilder urlBuilder(SFSCustomUrl(server.GetBaseUrl()), c_instanceId, c_namespace, handler); 336 337 // Will use a fake large product name to produce a large url 338 const std::string largeProductName(90000, 'a'); 339 340 // Register the products 341 server.RegisterProduct(largeProductName, c_version); 342 343 // Url produces: 414 URI Too Long 344 std::string out; 345 REQUIRE_THROWS_CODE_MSG(connection->Get(urlBuilder.GetSpecificVersionUrl(largeProductName, c_version)), 346 HttpUnexpected, 347 "Unexpected HTTP code 414"); 348 } 349 350 TEST("Testing a response over the limit fails the operation") 351 { 352 ReportingHandler handler; 353 handler.SetLoggingCallback(LogCallbackToTest); 354 355 test::MockWebServer server; 356 CurlConnectionManager connectionManager(handler); 357 auto connection = connectionManager.MakeConnection({}); 358 SFSUrlBuilder urlBuilder(SFSCustomUrl(server.GetBaseUrl()), c_instanceId, c_namespace, handler); 359 360 // Will use a fake large product name to produce a response over the limit of 100k characters 361 const std::string largeProductName(90000, 'a'); 362 const std::string overLimitProductName(1000000, 'a'); 363 364 // Register the products 365 server.RegisterProduct(largeProductName, c_version); 366 server.RegisterProduct(overLimitProductName, c_version); 367 368 // Using GetLatestVersionBatch api since the product name is in the body and not in the url, to avoid a 414 error 369 // like on the test above 370 const std::string url = urlBuilder.GetLatestVersionBatchUrl(); 371 372 // Large one works 373 json body = {{{"TargetingAttributes", {}}, {"Product", largeProductName}}}; 374 REQUIRE_NOTHROW(connection->Post(url, body.dump())); 375 376 // Going over the limit fails with a message like "client returned ERROR on write of 16384 bytes" 377 body[0]["Product"] = overLimitProductName; 378 REQUIRE_THROWS_CODE_MSG_MATCHES(connection->Post(url, body.dump()), 379 ConnectionUnexpectedError, 380 Catch::Matchers::ContainsSubstring("client returned ERROR on write of")); 381 } 382 383 TEST("Testing MS-CV is sent to server") 384 { 385 test::MockWebServer server; 386 ReportingHandler handler; 387 handler.SetLoggingCallback(LogCallbackToTest); 388 CurlConnectionManager connectionManager(handler); 389 390 const std::string cv = "aaaaaaaaaaaaaaaa.1"; 391 ConnectionConfig config; 392 config.baseCV = cv; 393 auto connection = connectionManager.MakeConnection(config); 394 SFSUrlBuilder urlBuilder(SFSCustomUrl(server.GetBaseUrl()), c_instanceId, c_namespace, handler); 395 396 const std::string url = urlBuilder.GetSpecificVersionUrl(c_productName, c_version); 397 398 server.RegisterProduct(c_productName, c_version); 399 400 INFO("First value ends with .0"); 401 server.RegisterExpectedRequestHeader(HttpHeader::MSCV, cv + ".0"); 402 connection->Get(url); 403 404 INFO("Value gets incremented on subsequent requests"); 405 server.RegisterExpectedRequestHeader(HttpHeader::MSCV, cv + ".1"); 406 connection->Get(url); 407 408 server.RegisterExpectedRequestHeader(HttpHeader::MSCV, cv + ".2"); 409 connection->Get(url); 410 } 411 412 TEST("Testing retry behavior") 413 { 414 if (!AreTestOverridesAllowed()) 415 { 416 INFO("Skipping. Test overrides not enabled"); 417 return; 418 } 419 420 ReportingHandler handler; 421 handler.SetLoggingCallback(LogCallbackToTest); 422 423 MockWebServer server; 424 CurlConnectionManager connectionManager(handler); 425 SFSUrlBuilder urlBuilder(SFSCustomUrl(server.GetBaseUrl()), c_instanceId, c_namespace, handler); 426 427 server.RegisterProduct(c_productName, c_version); 428 const std::string url = urlBuilder.GetSpecificVersionUrl(c_productName, c_version); 429 430 SECTION("Test exponential backoff") 431 { 432 auto connection = connectionManager.MakeConnection({}); 433 434 INFO("Sets the retry delay to 50ms to speed up the test"); 435 ScopedTestOverride override(TestOverride::BaseRetryDelayMs, 50); 436 437 const int retriableError = 503; // ServerBusy 438 439 auto RunTimedGet = [&](bool success = true) -> long long { 440 std::string out; 441 auto begin = steady_clock::now(); 442 if (success) 443 { 444 REQUIRE_NOTHROW(out = connection->Get(url)); 445 REQUIRE_FALSE(out.empty()); 446 } 447 else 448 { 449 REQUIRE_THROWS_CODE(out = connection->Get(url), HttpServiceNotAvailable); 450 REQUIRE(out.empty()); 451 } 452 auto end = steady_clock::now(); 453 return duration_cast<milliseconds>(end - begin).count(); 454 }; 455 456 long long allowedTimeDeviation = 200LL; 457 std::queue<HttpCode> forcedHttpErrors({retriableError}); 458 SECTION("Should take at least 50ms with a single retriable error") 459 { 460 server.SetForcedHttpErrors(forcedHttpErrors); 461 const auto time = RunTimedGet(); 462 REQUIRE(time >= 50LL); 463 REQUIRE(time < 50LL + allowedTimeDeviation); 464 } 465 466 forcedHttpErrors.push(retriableError); 467 SECTION("Should take at least 150ms (50ms + 2*50ms) with two retriable errors") 468 { 469 server.SetForcedHttpErrors(forcedHttpErrors); 470 const auto time = RunTimedGet(); 471 REQUIRE(time >= 150LL); 472 REQUIRE(time < 150LL + allowedTimeDeviation); 473 } 474 475 forcedHttpErrors.push(retriableError); 476 SECTION("Should take at least 300ms (50ms + 2*50ms + 3*50ms) with three retriable errors") 477 { 478 server.SetForcedHttpErrors(forcedHttpErrors); 479 const auto time = RunTimedGet(); 480 REQUIRE(time >= 300LL); 481 REQUIRE(time < 300LL + allowedTimeDeviation); 482 } 483 484 forcedHttpErrors.push(retriableError); 485 SECTION("Should take at least 300ms (50ms + 2*50ms + 3*50ms) with four retriable errors, but fail") 486 { 487 server.SetForcedHttpErrors(forcedHttpErrors); 488 const auto time = RunTimedGet(false /*success*/); 489 REQUIRE(time >= 300LL); 490 REQUIRE(time < 300LL + allowedTimeDeviation); 491 } 492 } 493 494 SECTION("Test retriable errors with Retry-After headers") 495 { 496 auto connection = connectionManager.MakeConnection({}); 497 498 INFO("Sets the retry delay to 200ms to speed up the test"); 499 ScopedTestOverride override(TestOverride::BaseRetryDelayMs, 200); 500 501 const int retriableError = 503; // ServerBusy 502 const int retriableError2 = 502; // BadGateway 503 504 auto RunTimedGet = [&]() -> long long { 505 std::string out; 506 auto begin = steady_clock::now(); 507 REQUIRE_NOTHROW(out = connection->Get(url)); 508 REQUIRE_FALSE(out.empty()); 509 auto end = steady_clock::now(); 510 return duration_cast<milliseconds>(end - begin).count(); 511 }; 512 513 auto testForRetryAfterValue = [&](const std::string& retryAfterValue) -> void { 514 INFO("retryAfterValue = " << retryAfterValue); 515 516 std::unordered_map<HttpCode, HeaderMap> headersByCode; 517 headersByCode[retriableError] = {{"Retry-After", retryAfterValue}}; 518 server.SetResponseHeaders(headersByCode); 519 520 long long allowedTimeDeviation = 200LL; 521 std::queue<HttpCode> forcedHttpErrors({retriableError}); 522 SECTION("Should take at least 2000ms with a single retriable error, with 2s in Retry-After") 523 { 524 server.SetForcedHttpErrors(forcedHttpErrors); 525 const auto time = RunTimedGet(); 526 REQUIRE(time >= 2000LL); 527 REQUIRE(time < 2000LL + allowedTimeDeviation); 528 } 529 530 SECTION( 531 "Should take at least 2000ms + 200ms*2 (exponential backoff) with two retriable errors, with 2s in Retry-After of the first error") 532 { 533 forcedHttpErrors.push(retriableError2); 534 server.SetForcedHttpErrors(forcedHttpErrors); 535 const auto time = RunTimedGet(); 536 REQUIRE(time >= 2400LL); 537 REQUIRE(time < 2400LL + allowedTimeDeviation); 538 } 539 }; 540 541 SECTION("Using seconds") 542 { 543 testForRetryAfterValue("2"); // 2s delay 544 } 545 SECTION("Using date") 546 { 547 const auto time = std::chrono::system_clock::now() + std::chrono::seconds(2); 548 testForRetryAfterValue(TimestampToHttpDateString(time)); 549 } 550 } 551 552 SECTION("Test maxRetries") 553 { 554 INFO("Sets the retry delay to 1ms to speed up the test"); 555 ScopedTestOverride override(TestOverride::BaseRetryDelayMs, 1); 556 557 const int retriableError = 503; // ServerBusy 558 559 SECTION("Should pass with 3 errors") 560 { 561 auto connection = connectionManager.MakeConnection({}); 562 server.SetForcedHttpErrors(std::queue<HttpCode>({retriableError, retriableError, retriableError})); 563 REQUIRE_NOTHROW(connection->Get(url)); 564 } 565 566 SECTION("Should fail with 4 errors") 567 { 568 auto connection = connectionManager.MakeConnection({}); 569 server.SetForcedHttpErrors( 570 std::queue<HttpCode>({retriableError, retriableError, retriableError, retriableError})); 571 REQUIRE_THROWS_CODE(connection->Get(url), HttpServiceNotAvailable); 572 } 573 574 SECTION("Reducing retries to 2") 575 { 576 ConnectionConfig config; 577 config.maxRetries = 2; 578 auto connection = connectionManager.MakeConnection(config); 579 580 SECTION("Should pass with 2 errors") 581 { 582 server.SetForcedHttpErrors(std::queue<HttpCode>({retriableError, retriableError})); 583 REQUIRE_NOTHROW(connection->Get(url)); 584 } 585 586 SECTION("Should fail with 3 errors") 587 { 588 server.SetForcedHttpErrors(std::queue<HttpCode>({retriableError, retriableError, retriableError})); 589 REQUIRE_THROWS_CODE(connection->Get(url), HttpServiceNotAvailable); 590 } 591 } 592 593 SECTION("Reducing retries to 0") 594 { 595 ConnectionConfig config; 596 config.maxRetries = 0; 597 auto connection = connectionManager.MakeConnection(config); 598 599 SECTION("Should pass with no errors") 600 { 601 REQUIRE_NOTHROW(connection->Get(url)); 602 } 603 604 SECTION("Should fail with 1 error") 605 { 606 server.SetForcedHttpErrors(std::queue<HttpCode>({retriableError})); 607 REQUIRE_THROWS_CODE(connection->Get(url), HttpServiceNotAvailable); 608 } 609 } 610 } 611 }