YamlParser.cpp (27849B)
1 // Copyright (c) Microsoft Corporation. 2 // Licensed under the MIT License. 3 #include "pch.h" 4 #include "AppInstallerSHA256.h" 5 #include "winget/Yaml.h" 6 #include "winget/ManifestSchemaValidation.h" 7 #include "winget/ManifestYamlPopulator.h" 8 #include "winget/ManifestYamlParser.h" 9 10 namespace AppInstaller::Manifest::YamlParser 11 { 12 namespace 13 { 14 // Basic V1 manifest required fields check for later manifest consistency check 15 void ValidateV1ManifestInput(const YamlManifestInfo& entry) 16 { 17 std::vector<ValidationError> errors; 18 19 if (!entry.Root.IsMap()) 20 { 21 THROW_EXCEPTION_MSG(ManifestException(APPINSTALLER_CLI_ERROR_INVALID_MANIFEST), "The manifest does not contain a valid root. File: %hs", entry.FileName.c_str()); 22 } 23 24 if (!entry.Root["PackageIdentifier"]) 25 { 26 errors.emplace_back(ValidationError::MessageContextWithFile( 27 ManifestError::RequiredFieldMissing, "PackageIdentifier", entry.FileName)); 28 } 29 30 if (!entry.Root["PackageVersion"]) 31 { 32 errors.emplace_back(ValidationError::MessageContextWithFile( 33 ManifestError::RequiredFieldMissing, "PackageVersion", entry.FileName)); 34 } 35 36 if (!entry.Root["ManifestVersion"]) 37 { 38 errors.emplace_back(ValidationError::MessageContextWithFile( 39 ManifestError::RequiredFieldMissing, "ManifestVersion", entry.FileName)); 40 } 41 42 if (!entry.Root["ManifestType"]) 43 { 44 errors.emplace_back(ValidationError::MessageContextWithFile( 45 ManifestError::InconsistentMultiFileManifestFieldValue, "ManifestType", entry.FileName)); 46 } 47 else 48 { 49 auto manifestType = ConvertToManifestTypeEnum(entry.Root["ManifestType"].as<std::string>()); 50 51 switch (manifestType) 52 { 53 case ManifestTypeEnum::Version: 54 if (!entry.Root["DefaultLocale"]) 55 { 56 errors.emplace_back(ValidationError::MessageContextWithFile( 57 ManifestError::RequiredFieldMissing, "DefaultLocale", entry.FileName)); 58 } 59 break; 60 case ManifestTypeEnum::Singleton: 61 case ManifestTypeEnum::Locale: 62 case ManifestTypeEnum::DefaultLocale: 63 if (!entry.Root["PackageLocale"]) 64 { 65 errors.emplace_back(ValidationError::MessageContextWithFile( 66 ManifestError::RequiredFieldMissing, "PackageLocale", entry.FileName)); 67 } 68 break; 69 } 70 } 71 72 if (!errors.empty()) 73 { 74 ManifestException ex{ std::move(errors) }; 75 THROW_EXCEPTION(ex); 76 } 77 } 78 79 // Input validations: 80 // - Determine manifest version 81 // - Check multi file manifest input integrity 82 // - All manifests use same PackageIdentifier, PackageVersion, ManifestVersion 83 // - All required types exist and exist only once. i.e. version, installer, defaultLocale 84 // - No duplicate locales across manifests 85 // - DefaultLocale matches in version manifest and defaultLocale manifest 86 // - Validate manifest type correctness 87 // - Allowed file type in multi file manifest: version, installer, defaultLocale, locale 88 // - Allowed file type in single file manifest: preview manifest, merged and singleton 89 ManifestVer ValidateInput(std::vector<YamlManifestInfo>& input, ManifestValidateOption validateOption) 90 { 91 std::vector<ValidationError> errors; 92 93 std::string manifestVersionStr; 94 ManifestVer manifestVersion; 95 ManifestVer ManifestVersionV1{ s_ManifestVersionV1 }; 96 bool isMultifileManifest = input.size() > 1; 97 98 // Use the first manifest doc to determine ManifestVersion, there'll be checks for manifest version consistency later 99 auto& firstYamlManifest = input[0]; 100 if (!firstYamlManifest.Root.IsMap()) 101 { 102 THROW_EXCEPTION_MSG(ManifestException(APPINSTALLER_CLI_ERROR_INVALID_MANIFEST), "The manifest does not contain a valid root. File: %hs", firstYamlManifest.FileName.c_str()); 103 } 104 105 if (firstYamlManifest.Root["ManifestVersion"sv]) 106 { 107 manifestVersionStr = firstYamlManifest.Root["ManifestVersion"sv].as<std::string>(); 108 } 109 else 110 { 111 manifestVersionStr = s_DefaultManifestVersion; 112 } 113 manifestVersion = ManifestVer{ manifestVersionStr }; 114 115 // Check max supported version 116 if (manifestVersion.Major() > s_MaxSupportedMajorVersion) 117 { 118 THROW_EXCEPTION_MSG(ManifestException(APPINSTALLER_CLI_ERROR_UNSUPPORTED_MANIFESTVERSION), "Unsupported ManifestVersion: %hs", manifestVersion.ToString().c_str()); 119 } 120 121 // Preview manifest validations 122 if (manifestVersion < ManifestVersionV1) 123 { 124 // multi file manifest is only supported starting ManifestVersion 1.0.0 125 if (isMultifileManifest) 126 { 127 THROW_EXCEPTION_MSG(ManifestException(APPINSTALLER_CLI_ERROR_INVALID_MANIFEST), "Preview manifest does not support multi file manifest format."); 128 } 129 130 firstYamlManifest.ManifestType = ManifestTypeEnum::Preview; 131 } 132 // V1 manifest validations 133 else 134 { 135 // Check required fields used by later consistency check for better error message instead of 136 // Field Type Not Match error. 137 for (auto const& entry : input) 138 { 139 ValidateV1ManifestInput(entry); 140 } 141 142 if (isMultifileManifest) 143 { 144 // Populates the PackageIdentifier and PackageVersion from first doc for later consistency check 145 std::string packageId = firstYamlManifest.Root["PackageIdentifier"].as<std::string>(); 146 std::string packageVersion = firstYamlManifest.Root["PackageVersion"].as<std::string>(); 147 148 std::set<std::string> localesSet; 149 150 bool isVersionManifestFound = false; 151 bool isInstallerManifestFound = false; 152 bool isDefaultLocaleManifestFound = false; 153 bool isShadowManifestFound = false; 154 std::string defaultLocaleFromVersionManifest; 155 std::string defaultLocaleFromDefaultLocaleManifest; 156 157 for (auto& entry : input) 158 { 159 std::string localPackageId = entry.Root["PackageIdentifier"].as<std::string>(); 160 if (localPackageId != packageId) 161 { 162 errors.emplace_back(ValidationError::MessageContextValueWithFile( 163 ManifestError::InconsistentMultiFileManifestFieldValue, "PackageIdentifier", localPackageId, entry.FileName)); 164 } 165 166 std::string localPackageVersion = entry.Root["PackageVersion"].as<std::string>(); 167 if (localPackageVersion != packageVersion) 168 { 169 errors.emplace_back(ValidationError::MessageContextValueWithFile( 170 ManifestError::InconsistentMultiFileManifestFieldValue, "PackageVersion", localPackageVersion, entry.FileName)); 171 } 172 173 std::string localManifestVersion = entry.Root["ManifestVersion"].as<std::string>(); 174 if (localManifestVersion != manifestVersionStr) 175 { 176 errors.emplace_back(ValidationError::MessageContextValueWithFile( 177 ManifestError::InconsistentMultiFileManifestFieldValue, "ManifestVersion", localManifestVersion, entry.FileName)); 178 } 179 180 std::string manifestTypeStr = entry.Root["ManifestType"sv].as<std::string>(); 181 ManifestTypeEnum manifestType = ConvertToManifestTypeEnum(manifestTypeStr); 182 entry.ManifestType = manifestType; 183 184 switch (manifestType) 185 { 186 case ManifestTypeEnum::Version: 187 if (isVersionManifestFound) 188 { 189 errors.emplace_back(ValidationError::MessageContextValueWithFile( 190 ManifestError::DuplicateMultiFileManifestType, "ManifestType", manifestTypeStr, entry.FileName)); 191 } 192 else 193 { 194 isVersionManifestFound = true; 195 defaultLocaleFromVersionManifest = entry.Root["DefaultLocale"sv].as<std::string>(); 196 } 197 break; 198 case ManifestTypeEnum::Installer: 199 if (isInstallerManifestFound) 200 { 201 errors.emplace_back(ValidationError::MessageContextValueWithFile( 202 ManifestError::DuplicateMultiFileManifestType, "ManifestType", manifestTypeStr, entry.FileName)); 203 } 204 else 205 { 206 isInstallerManifestFound = true; 207 } 208 break; 209 case ManifestTypeEnum::DefaultLocale: 210 if (isDefaultLocaleManifestFound) 211 { 212 errors.emplace_back(ValidationError::MessageContextValueWithFile( 213 ManifestError::DuplicateMultiFileManifestType, "ManifestType", manifestTypeStr, entry.FileName)); 214 } 215 else 216 { 217 isDefaultLocaleManifestFound = true; 218 auto packageLocale = entry.Root["PackageLocale"sv].as<std::string>(); 219 defaultLocaleFromDefaultLocaleManifest = packageLocale; 220 221 if (localesSet.find(packageLocale) != localesSet.end()) 222 { 223 errors.emplace_back(ValidationError::MessageContextValueWithFile( 224 ManifestError::DuplicateMultiFileManifestLocale, "PackageLocale", packageLocale, entry.FileName)); 225 } 226 else 227 { 228 localesSet.insert(packageLocale); 229 } 230 } 231 break; 232 case ManifestTypeEnum::Locale: 233 { 234 auto packageLocale = entry.Root["PackageLocale"sv].as<std::string>(); 235 if (localesSet.find(packageLocale) != localesSet.end()) 236 { 237 errors.emplace_back(ValidationError::MessageContextValueWithFile( 238 ManifestError::DuplicateMultiFileManifestLocale, "PackageLocale", packageLocale, entry.FileName)); 239 } 240 else 241 { 242 localesSet.insert(packageLocale); 243 } 244 break; 245 } 246 case ManifestTypeEnum::Shadow: 247 { 248 if (!validateOption.AllowShadowManifest) 249 { 250 errors.emplace_back(ValidationError::MessageContextValueWithFile( 251 ManifestError::ShadowManifestNotAllowed, "ManifestType", manifestTypeStr, entry.FileName)); 252 } 253 else if (isShadowManifestFound) 254 { 255 errors.emplace_back(ValidationError::MessageContextValueWithFile( 256 ManifestError::DuplicateMultiFileManifestType, "ManifestType", manifestTypeStr, entry.FileName)); 257 } 258 else 259 { 260 isShadowManifestFound = true; 261 } 262 break; 263 } 264 default: 265 errors.emplace_back(ValidationError::MessageContextValueWithFile( 266 ManifestError::UnsupportedMultiFileManifestType, "ManifestType", manifestTypeStr, entry.FileName)); 267 } 268 } 269 270 if (isVersionManifestFound && isDefaultLocaleManifestFound && defaultLocaleFromDefaultLocaleManifest != defaultLocaleFromVersionManifest) 271 { 272 errors.emplace_back(ManifestError::InconsistentMultiFileManifestDefaultLocale); 273 } 274 275 if (!validateOption.SchemaValidationOnly && !(isVersionManifestFound && isInstallerManifestFound && isDefaultLocaleManifestFound)) 276 { 277 errors.emplace_back(ManifestError::IncompleteMultiFileManifest); 278 } 279 } 280 else 281 { 282 std::string manifestTypeStr = firstYamlManifest.Root["ManifestType"sv].as<std::string>(); 283 ManifestTypeEnum manifestType = ConvertToManifestTypeEnum(manifestTypeStr); 284 firstYamlManifest.ManifestType = manifestType; 285 286 if (validateOption.FullValidation && manifestType == ManifestTypeEnum::Merged) 287 { 288 errors.emplace_back(ValidationError::MessageContextValueWithFile(ManifestError::FieldValueNotSupported, "ManifestType", manifestTypeStr, firstYamlManifest.FileName)); 289 } 290 291 if (!validateOption.SchemaValidationOnly && manifestType != ManifestTypeEnum::Merged && manifestType != ManifestTypeEnum::Singleton) 292 { 293 errors.emplace_back(ValidationError::MessageWithFile(ManifestError::IncompleteMultiFileManifest, firstYamlManifest.FileName)); 294 } 295 } 296 } 297 298 if (!errors.empty()) 299 { 300 ManifestException ex{ std::move(errors) }; 301 THROW_EXCEPTION(ex); 302 } 303 304 return manifestVersion; 305 } 306 307 // Find a unique required manifest from the input in multi manifest case 308 const YAML::Node& FindUniqueRequiredDocFromMultiFileManifest(const std::vector<YamlManifestInfo>& input, ManifestTypeEnum manifestType) 309 { 310 auto iter = std::find_if(input.begin(), input.end(), 311 [=](auto const& s) 312 { 313 return s.ManifestType == manifestType; 314 }); 315 316 THROW_HR_IF(E_UNEXPECTED, iter == input.end()); 317 318 return iter->Root; 319 } 320 321 std::optional<YAML::Node> FindUniqueOptionalDocFromMultiFileManifest(std::vector<YamlManifestInfo>& input, ManifestTypeEnum manifestType) 322 { 323 auto iter = std::find_if(input.begin(), input.end(), 324 [=](auto const& s) 325 { 326 return s.ManifestType == manifestType; 327 }); 328 329 if (iter != input.end()) 330 { 331 return iter->Root; 332 } 333 334 return {}; 335 } 336 337 // Merge one manifest file to the final merged manifest, basically copying the mapping but excluding certain common fields 338 void MergeOneManifestToMultiFileManifest(const YAML::Node& input, YAML::Node& destination) 339 { 340 THROW_HR_IF(E_UNEXPECTED, !input.IsMap()); 341 THROW_HR_IF(E_UNEXPECTED, !destination.IsMap()); 342 343 const std::vector<std::string> FieldsToIgnore = { "PackageIdentifier", "PackageVersion", "ManifestType", "ManifestVersion" }; 344 345 for (auto const& keyValuePair : input.Mapping()) 346 { 347 // We only support string type as key in our manifest 348 if (std::find(FieldsToIgnore.begin(), FieldsToIgnore.end(), keyValuePair.first.as<std::string>()) == FieldsToIgnore.end()) 349 { 350 YAML::Node key = keyValuePair.first; 351 YAML::Node value = keyValuePair.second; 352 destination.AddMappingNode(std::move(key), std::move(value)); 353 } 354 } 355 } 356 357 YAML::Node MergeMultiFileManifest(const std::vector<YamlManifestInfo>& input) 358 { 359 // Starts with installer manifest 360 YAML::Node result = FindUniqueRequiredDocFromMultiFileManifest(input, ManifestTypeEnum::Installer); 361 362 // Copy default locale manifest content into manifest root 363 YAML::Node defaultLocaleManifest = FindUniqueRequiredDocFromMultiFileManifest(input, ManifestTypeEnum::DefaultLocale); 364 MergeOneManifestToMultiFileManifest(defaultLocaleManifest, result); 365 366 // Copy additional locale manifests 367 YAML::Node localizations{ YAML::Node::Type::Sequence, "", YAML::Mark() }; 368 for (const auto& entry : input) 369 { 370 if (entry.ManifestType == ManifestTypeEnum::Locale) 371 { 372 YAML::Node localization{ YAML::Node::Type::Mapping, "", YAML::Mark() }; 373 MergeOneManifestToMultiFileManifest(entry.Root, localization); 374 localizations.AddSequenceNode(std::move(localization)); 375 } 376 } 377 378 if (localizations.size() > 0) 379 { 380 YAML::Node key{ YAML::Node::Type::Scalar, "", YAML::Mark() }; 381 key.SetScalar("Localization"); 382 result.AddMappingNode(std::move(key), std::move(localizations)); 383 } 384 385 result["ManifestType"sv].SetScalar("merged"); 386 387 return result; 388 } 389 390 void EmitYamlNode(const YAML::Node& input, YAML::Emitter& emitter) 391 { 392 if (input.IsMap()) 393 { 394 emitter << YAML::BeginMap; 395 for (auto const& keyValuePair : input.Mapping()) 396 { 397 emitter << YAML::Key; 398 EmitYamlNode(keyValuePair.first, emitter); 399 emitter << YAML::Value; 400 EmitYamlNode(keyValuePair.second, emitter); 401 } 402 emitter << YAML::EndMap; 403 } 404 else if (input.IsSequence()) 405 { 406 emitter << YAML::BeginSeq; 407 for (auto const& value : input.Sequence()) 408 { 409 EmitYamlNode(value, emitter); 410 } 411 emitter << YAML::EndSeq; 412 } 413 else if (input.IsScalar()) 414 { 415 emitter << input.as<std::string>(); 416 } 417 else if (input.IsNull()) 418 { 419 emitter << ""; 420 } 421 else 422 { 423 THROW_HR(E_UNEXPECTED); 424 } 425 } 426 427 void OutputYamlDoc(const YAML::Node& input, const std::filesystem::path& out) 428 { 429 THROW_HR_IF(E_UNEXPECTED, !input.IsMap()); 430 431 YAML::Emitter emitter; 432 EmitYamlNode(input, emitter); 433 434 std::filesystem::create_directories(out.parent_path()); 435 std::ofstream outFileStream(out); 436 emitter.Emit(outFileStream); 437 outFileStream.close(); 438 } 439 440 std::vector<ValidationError> ParseManifestImpl( 441 std::vector<YamlManifestInfo>& input, 442 Manifest& manifest, 443 const std::filesystem::path& mergedManifestPath, 444 ManifestValidateOption validateOption) 445 { 446 THROW_HR_IF_MSG(E_INVALIDARG, input.size() == 0, "No manifest file found"); 447 THROW_HR_IF_MSG(E_INVALIDARG, validateOption.SchemaValidationOnly && !mergedManifestPath.empty(), "Manifest cannot be merged if only schema validation is performed"); 448 THROW_HR_IF_MSG(E_INVALIDARG, input.size() == 1 && !mergedManifestPath.empty(), "Manifest cannot be merged from a single manifest"); 449 450 auto manifestVersion = ValidateInput(input, validateOption); 451 452 std::vector<ValidationError> resultErrors; 453 454 if (validateOption.FullValidation || validateOption.SchemaValidationOnly) 455 { 456 resultErrors = ValidateAgainstSchema(input, manifestVersion); 457 } 458 459 if (validateOption.SchemaValidationOnly) 460 { 461 return resultErrors; 462 } 463 464 // Merge manifests in multi file manifest case 465 bool isMultiFile = input.size() > 1; 466 YAML::Node& manifestDoc = input[0].Root; 467 if (isMultiFile) 468 { 469 manifestDoc = MergeMultiFileManifest(input); 470 } 471 472 auto shadowNode = isMultiFile ? FindUniqueOptionalDocFromMultiFileManifest(input, ManifestTypeEnum::Shadow) : std::optional<YAML::Node>{}; 473 474 auto errors = ManifestYamlPopulator::PopulateManifest(manifestDoc, manifest, manifestVersion, validateOption, shadowNode); 475 std::move(errors.begin(), errors.end(), std::inserter(resultErrors, resultErrors.end())); 476 477 // Extra semantic validations after basic validation and field population 478 if (validateOption.FullValidation) 479 { 480 errors = ValidateManifest(manifest); 481 std::move(errors.begin(), errors.end(), std::inserter(resultErrors, resultErrors.end())); 482 483 // Validate the schema header for manifest version 1.7 and above 484 if (manifestVersion >= ManifestVer{ s_ManifestVersionV1_7 }) 485 { 486 // Validate the schema header. 487 errors = ValidateYamlManifestsSchemaHeader(input, manifestVersion, validateOption.SchemaHeaderValidationAsWarning); 488 std::move(errors.begin(), errors.end(), std::inserter(resultErrors, resultErrors.end())); 489 } 490 } 491 492 if (validateOption.InstallerValidation) 493 { 494 errors = ValidateManifestInstallers(manifest); 495 std::move(errors.begin(), errors.end(), std::inserter(resultErrors, resultErrors.end())); 496 } 497 498 // Output merged manifest if requested 499 if (!mergedManifestPath.empty()) 500 { 501 OutputYamlDoc(manifestDoc, mergedManifestPath); 502 } 503 504 // If there is only one input file, use its hash for the stream 505 if (input.size() == 1) 506 { 507 manifest.StreamSha256 = std::move(input[0].StreamSha256); 508 } 509 510 return resultErrors; 511 } 512 } 513 514 Manifest CreateFromPath( 515 const std::filesystem::path& inputPath, 516 ManifestValidateOption validateOption, 517 const std::filesystem::path& mergedManifestPath) 518 { 519 std::vector<YamlManifestInfo> docList; 520 521 try 522 { 523 if (std::filesystem::is_directory(inputPath)) 524 { 525 for (const auto& file : std::filesystem::directory_iterator(inputPath)) 526 { 527 THROW_HR_IF_MSG(HRESULT_FROM_WIN32(ERROR_DIRECTORY_NOT_SUPPORTED), std::filesystem::is_directory(file.path()), "Subdirectory not supported in manifest path"); 528 529 YamlManifestInfo manifestInfo; 530 YAML::Document doc = YAML::LoadDocument(file.path()); 531 manifestInfo.DocumentSchemaHeader = doc.GetSchemaHeader(); 532 manifestInfo.Root = std::move(doc).GetRoot(); 533 manifestInfo.FileName = file.path().filename().u8string(); 534 docList.emplace_back(std::move(manifestInfo)); 535 } 536 } 537 else 538 { 539 YamlManifestInfo manifestInfo; 540 YAML::Document doc = YAML::LoadDocument(inputPath, manifestInfo.StreamSha256); 541 manifestInfo.DocumentSchemaHeader = doc.GetSchemaHeader(); 542 manifestInfo.Root = std::move(doc).GetRoot(); 543 manifestInfo.FileName = inputPath.filename().u8string(); 544 docList.emplace_back(std::move(manifestInfo)); 545 } 546 } 547 catch (const std::exception& e) 548 { 549 THROW_EXCEPTION_MSG(ManifestException(), "%hs", e.what()); 550 } 551 552 return ParseManifest(docList, validateOption, mergedManifestPath); 553 } 554 555 Manifest Create( 556 const std::string& input, 557 ManifestValidateOption validateOption, 558 const std::filesystem::path& mergedManifestPath) 559 { 560 std::vector<YamlManifestInfo> docList; 561 562 try 563 { 564 YamlManifestInfo manifestInfo; 565 YAML::Document doc = YAML::LoadDocument(input); 566 manifestInfo.DocumentSchemaHeader = doc.GetSchemaHeader(); 567 manifestInfo.Root = std::move(doc).GetRoot(); 568 docList.emplace_back(std::move(manifestInfo)); 569 } 570 catch (const std::exception& e) 571 { 572 THROW_EXCEPTION_MSG(ManifestException(), "%hs", e.what()); 573 } 574 575 return ParseManifest(docList, validateOption, mergedManifestPath); 576 } 577 578 Manifest ParseManifest( 579 std::vector<YamlManifestInfo>& input, 580 ManifestValidateOption validateOption, 581 const std::filesystem::path& mergedManifestPath) 582 { 583 Manifest manifest; 584 std::vector<ValidationError> errors; 585 586 try 587 { 588 errors = ParseManifestImpl(input, manifest, mergedManifestPath, validateOption); 589 } 590 catch (const ManifestException&) 591 { 592 // Prevent ManifestException from being wrapped in another ManifestException 593 throw; 594 } 595 catch (const std::exception& e) 596 { 597 THROW_EXCEPTION_MSG(ManifestException(), "%hs", e.what()); 598 } 599 600 if (!errors.empty()) 601 { 602 ManifestException ex{ std::move(errors) }; 603 604 if (validateOption.ThrowOnWarning || !ex.IsWarningOnly()) 605 { 606 THROW_EXCEPTION(ex); 607 } 608 } 609 610 return manifest; 611 } 612 }