commit 015f0e9d9d6fd8d0d40e51dce264cd7744886c7b parent c555273a04843523cd248657caf92b22168483c5 Author: yao-msft <50888816+yao-msft@users.noreply.github.com> Date: Wed, 26 Jun 2024 10:23:08 -0700 Component Governance fix: Update libyaml (#4583) Component governance fix: update libyaml Diffstat:
21 files changed, 303 insertions(+), 36 deletions(-)
diff --git a/cgmanifest.json b/cgmanifest.json @@ -15,7 +15,7 @@ "type": "git", "git": { "repositoryUrl": "https://github.com/yaml/libyaml.git", - "commitHash": "2c891fc7a770e8ba2fec34fc6b545c672beb37e6" + "commitHash": "840b65c40675e2d06bf40405ad3f12dec7f35923" } } }, diff --git a/src/YamlCppLib/libyaml/.github/workflows/dist.yaml b/src/YamlCppLib/libyaml/.github/workflows/dist.yaml @@ -0,0 +1,28 @@ +name: dist + +on: + push: + branches: [ release/* ] + +jobs: + dist: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - run: env | sort + + - name: Get image + run: | + time docker pull yamlio/libyaml-dev + docker images | grep libyaml + + - run: | + make -C pkg/docker libyaml-dist-ci + ls -l pkg/docker/output + + - uses: actions/upload-artifact@v2 + with: + name: release + path: pkg/docker/output/yaml-0* + diff --git a/src/YamlCppLib/libyaml/.github/workflows/main.yml b/src/YamlCppLib/libyaml/.github/workflows/main.yml @@ -10,6 +10,8 @@ jobs: build: + env: + CC: ${{ matrix.compiler }} runs-on: ${{ matrix.os }} strategy: matrix: @@ -21,16 +23,15 @@ jobs: - macOS-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - run: env | sort - name: Install software + if: ${{ matrix.os == 'macOS-latest' }} run: | - if [[ '${{ matrix.os }}' == macOS-latest ]]; then - brew install automake bash coreutils make - echo ::add-path::/usr/local/opt/coreutils/libexec/gnubin - echo ::add-path::/usr/local/opt/make/libexec/gnubin - fi + brew install automake bash coreutils make libtool + echo "/usr/local/opt/coreutils/libexec/gnubin" >> $GITHUB_PATH + echo "/usr/local/opt/make/libexec/gnubin" >> $GITHUB_PATH - name: Fetch branches run: | git config remote.origin.fetch +refs/heads/*:refs/remotes/origin/* @@ -48,12 +49,6 @@ jobs: - name: Compiler version run: ${{ matrix.compiler }} --version - env: - CC: ${{ matrix.compiler }} - run: cmake . - env: - CC: ${{ matrix.compiler }} - run: make - env: - CC: ${{ matrix.compiler }} - run: make test diff --git a/src/YamlCppLib/libyaml/.gitignore b/src/YamlCppLib/libyaml/.gitignore @@ -28,7 +28,6 @@ config.h* /configure stamp-h1 !config/config.h.in -/packaging/ /tests/run-dumper /tests/run-emitter /tests/run-emitter-test-suite diff --git a/src/YamlCppLib/libyaml/.makefile b/src/YamlCppLib/libyaml/.makefile @@ -23,6 +23,8 @@ MAKE_TARGETS := \ all \ all-am \ all-recursive \ + docker-build \ + docker-dist \ install \ test \ test-all \ diff --git a/src/YamlCppLib/libyaml/Makefile.am b/src/YamlCppLib/libyaml/Makefile.am @@ -17,7 +17,7 @@ maintainer-clean-local: -find ${builddir} -name Makefile.in -exec rm -f '{}' ';' distclean-local: - rm -fr tests/run-test-suite packaging + rm -fr tests/run-test-suite -git worktree prune .PHONY: bootstrap @@ -43,12 +43,9 @@ ifeq ($(LIBYAML_TEST_SUITE_RUN_REPO),$(LIBYAML_TEST_SUITE_RUN_REPO_DEFAULT)) git clone --branch $(LIBYAML_TEST_SUITE_RUN_BRANCH) $(LIBYAML_TEST_SUITE_RUN_REPO) $@ endif -packaging: - -git branch --track $@ origin/$@ - git worktree add --force $@ $@ +docker-build: + make -C pkg/docker build -docker-dist: packaging - make -C $</docker libyaml-dist +docker-dist: + make -C pkg/docker libyaml-dist -docker-test-pyyaml: packaging - make -C $</docker test-pyyaml diff --git a/src/YamlCppLib/libyaml/examples/strings.yaml b/src/YamlCppLib/libyaml/examples/strings.yaml @@ -1,4 +1,4 @@ -unqouted: string +unquoted: string literal-block: | This entire block of text will be the value of the 'literal-block' key, with line breaks being preserved. diff --git a/src/YamlCppLib/libyaml/include/yaml.h b/src/YamlCppLib/libyaml/include/yaml.h @@ -1095,7 +1095,7 @@ typedef struct yaml_parser_s { yaml_error_type_t error; /** Error description. */ const char *problem; - /** The byte about which the problem occured. */ + /** The byte about which the problem occurred. */ size_t problem_offset; /** The problematic value (@c -1 is none). */ int problem_value; @@ -1335,7 +1335,7 @@ yaml_parser_delete(yaml_parser_t *parser); * Set a string input. * * Note that the @a input pointer must be valid while the @a parser object - * exists. The application is responsible for destroing @a input after + * exists. The application is responsible for destroying @a input after * destroying the @a parser. * * @param[in,out] parser A parser object. @@ -1456,6 +1456,20 @@ yaml_parser_parse(yaml_parser_t *parser, yaml_event_t *event); YAML_DECLARE(int) yaml_parser_load(yaml_parser_t *parser, yaml_document_t *document); +/** + * Set the maximum depth of nesting. + * + * Default: 1000 + * + * Each nesting level increases the stack and the number of previous + * starting events that the parser has to check. + * + * @param[in] max The maximum number of allowed nested events + */ + +YAML_DECLARE(void) +yaml_set_max_nest_level(int max); + /** @} */ /** @@ -1734,7 +1748,7 @@ typedef struct yaml_emitter_s { size_t length; /** Does the scalar contain line breaks? */ int multiline; - /** Can the scalar be expessed in the flow plain style? */ + /** Can the scalar be expressed in the flow plain style? */ int flow_plain_allowed; /** Can the scalar be expressed in the block plain style? */ int block_plain_allowed; @@ -1950,7 +1964,7 @@ yaml_emitter_close(yaml_emitter_t *emitter); /** * Emit a YAML document. * - * The documen object may be generated using the yaml_parser_load() function + * The document object may be generated using the yaml_parser_load() function * or the yaml_document_initialize() function. The emitter takes the * responsibility for the document object and destroys its content after * it is emitted. The document object is destroyed even if the function fails. diff --git a/src/YamlCppLib/libyaml/pkg/ReadMe.md b/src/YamlCppLib/libyaml/pkg/ReadMe.md @@ -0,0 +1,77 @@ +# How to Make a `libyaml` Release + +## Versioning + +Update libyaml version in: +* announcement.msg +* Changes +* CMakeLists.txt + * `YAML_VERSION_MAJOR`, `YAML_VERSION_MINOR`, `YAML_VERSION_PATCH` +* .appveyor.yml +* configure.ac + * `YAML_MAJOR`, `YAML_MINOR`, `YAML_PATCH`, `YAML_RELEASE`, `YAML_CURRENT`, `YAML_REVISION` + +Commit and push everything to `release/0.x.y`. + +## Test and Create Release Archives + +### GitHub Actions Automation + +The github workflow: + + .github/workflows/dist.yaml + +will do this automatically for you. + +#### .github/workflows/dist.yaml + +This workflow will create release archives (`tar.gz` and `zip`). + +### Manually + +Make sure you have a clean git repository (no changed files). +The following process will clone your current git directory. + +This will need the docker image `yamlio/libyaml-dev`. +You can either pull it, or create it yourself: + + make docker-build + +### Create dist archives + +Run: + + make docker-dist + +It will run `make dist` in the container to create a tarball written to +`pkg/docker/output`. +It will also create a zipfile. + +## Update master + + git merge release/0.x.y + git tag -a 0.x.y + # <Editor opens> + # Paste the corresponding entry from the Changes file + # Look at an earlier release for how it should look like: + # git show 0.2.3 + git push origin master 0.x.y + +## Create a GitHub release + +Go to "Releases" and click on "Draft a new release". + +Fill in the tag you just created in the previous step. + +Fill in the release title: v0.x.y + +Paste the changelog into the description field. + +Upload the tar.gz and .zip file. + +You can "Save draft" and publish later, or directly click on "Publish release". + +## Update pyyaml.org + +See <https://github.com/yaml/pyyaml.org/blob/master/ReadMe.md>. + diff --git a/src/YamlCppLib/libyaml/pkg/docker/.gitignore b/src/YamlCppLib/libyaml/pkg/docker/.gitignore @@ -0,0 +1,3 @@ +output/* +!Makefile +!output/ReadMe diff --git a/src/YamlCppLib/libyaml/pkg/docker/Dockerfile b/src/YamlCppLib/libyaml/pkg/docker/Dockerfile @@ -0,0 +1,32 @@ +FROM ubuntu:18.04 + +RUN apt-get update \ + && apt-get install -y \ + automake \ + bison \ + build-essential \ + cmake \ + curl \ + doxygen \ + flex \ + git \ + less \ + libtool \ + python \ + vim \ + zip \ + && true + +# http://www.doxygen.nl/manual/install.html + +RUN curl https://sourceforge.net/projects/doxygen/files/rel-1.8.14/doxygen-1.8.14.src.tar.gz/download \ + -L -o /doxygen-1.8.14.src.tar.gz \ + && cd / \ + && tar -xvf doxygen-1.8.14.src.tar.gz \ + && cd doxygen-1.8.14 \ + && mkdir build \ + && cd build \ + && cmake -G "Unix Makefiles" .. \ + && make \ + && make install \ + && true diff --git a/src/YamlCppLib/libyaml/pkg/docker/Makefile b/src/YamlCppLib/libyaml/pkg/docker/Makefile @@ -0,0 +1,23 @@ +DOCKER_IMAGE ?= yamlio/libyaml-dev + +build: + docker build -t $(DOCKER_IMAGE) . + +run: build + docker run -it --rm $(DOCKER_IMAGE) bash + +prepare-git: + rm -rf output/libyaml.git + git clone ../../.git output/libyaml.git + +libyaml-dist: libyaml-dist-ci + +libyaml-dist-ci: prepare-git + docker run --rm -u $$(id -u) \ + -v"$$PWD/output:/output" \ + -v"$$PWD/scripts:/scripts" \ + $(DOCKER_IMAGE) /scripts/libyaml-dist.sh + +clean: + rm -rf output/libyaml.git + rm -rf output/yaml-*.* diff --git a/src/YamlCppLib/libyaml/pkg/docker/output/ReadMe b/src/YamlCppLib/libyaml/pkg/docker/output/ReadMe @@ -0,0 +1 @@ +Output directory for build files created by docker diff --git a/src/YamlCppLib/libyaml/pkg/docker/scripts/libyaml-dist.sh b/src/YamlCppLib/libyaml/pkg/docker/scripts/libyaml-dist.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -ex + +cp -r /output/libyaml.git /tmp/ +cd /tmp/libyaml.git +./bootstrap +./configure +make dist + +# get the tarball filename +tarballs=(yaml-*.tar.gz) +tarball=${tarballs[0]:?} +dirname=${tarball/.tar.gz/} + +# Copy to output dir +cp "$tarball" /output + +# Create zip archive +cd /tmp +cp "/output/$tarball" . +tar xvf "$tarball" +zip -r "/output/$dirname.zip" "$dirname" diff --git a/src/YamlCppLib/libyaml/src/parser.c b/src/YamlCppLib/libyaml/src/parser.c @@ -64,6 +64,8 @@ * Public API declarations. */ +int MAX_NESTING_LEVEL = 1000; + YAML_DECLARE(int) yaml_parser_parse(yaml_parser_t *parser, yaml_event_t *event); @@ -80,6 +82,10 @@ yaml_parser_set_parser_error_context(yaml_parser_t *parser, const char *context, yaml_mark_t context_mark, const char *problem, yaml_mark_t problem_mark); +static int +yaml_maximum_level_reached(yaml_parser_t *parser, + yaml_mark_t context_mark, yaml_mark_t problem_mark); + /* * State functions. */ @@ -162,6 +168,12 @@ static int yaml_parser_append_tag_directive(yaml_parser_t *parser, yaml_tag_directive_t value, int allow_duplicates, yaml_mark_t mark); +YAML_DECLARE(void) +yaml_set_max_nest_level(int max) +{ + MAX_NESTING_LEVEL = max; +} + /* * Get the next event. */ @@ -217,6 +229,14 @@ yaml_parser_set_parser_error_context(yaml_parser_t *parser, return 0; } +static int +yaml_maximum_level_reached(yaml_parser_t *parser, + yaml_mark_t context_mark, yaml_mark_t problem_mark) +{ + yaml_parser_set_parser_error_context(parser, + "while parsing", context_mark, "Maximum nesting level reached, set with yaml_set_max_nest_level())", problem_mark); + return 0; +} /* * State dispatcher. @@ -657,6 +677,10 @@ yaml_parser_parse_node(yaml_parser_t *parser, yaml_event_t *event, return 1; } else if (token->type == YAML_FLOW_SEQUENCE_START_TOKEN) { + if (!STACK_LIMIT(parser, parser->indents, MAX_NESTING_LEVEL - parser->flow_level)) { + yaml_maximum_level_reached(parser, start_mark, token->start_mark); + goto error; + } end_mark = token->end_mark; parser->state = YAML_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE; SEQUENCE_START_EVENT_INIT(*event, anchor, tag, implicit, @@ -664,6 +688,10 @@ yaml_parser_parse_node(yaml_parser_t *parser, yaml_event_t *event, return 1; } else if (token->type == YAML_FLOW_MAPPING_START_TOKEN) { + if (!STACK_LIMIT(parser, parser->indents, MAX_NESTING_LEVEL - parser->flow_level)) { + yaml_maximum_level_reached(parser, start_mark, token->start_mark); + goto error; + } end_mark = token->end_mark; parser->state = YAML_PARSE_FLOW_MAPPING_FIRST_KEY_STATE; MAPPING_START_EVENT_INIT(*event, anchor, tag, implicit, @@ -671,6 +699,10 @@ yaml_parser_parse_node(yaml_parser_t *parser, yaml_event_t *event, return 1; } else if (block && token->type == YAML_BLOCK_SEQUENCE_START_TOKEN) { + if (!STACK_LIMIT(parser, parser->indents, MAX_NESTING_LEVEL - parser->flow_level)) { + yaml_maximum_level_reached(parser, start_mark, token->start_mark); + goto error; + } end_mark = token->end_mark; parser->state = YAML_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE; SEQUENCE_START_EVENT_INIT(*event, anchor, tag, implicit, @@ -678,6 +710,10 @@ yaml_parser_parse_node(yaml_parser_t *parser, yaml_event_t *event, return 1; } else if (block && token->type == YAML_BLOCK_MAPPING_START_TOKEN) { + if (!STACK_LIMIT(parser, parser->indents, MAX_NESTING_LEVEL - parser->flow_level)) { + yaml_maximum_level_reached(parser, start_mark, token->start_mark); + goto error; + } end_mark = token->end_mark; parser->state = YAML_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE; MAPPING_START_EVENT_INIT(*event, anchor, tag, implicit, @@ -1022,6 +1058,11 @@ yaml_parser_parse_flow_sequence_entry_mapping_key(yaml_parser_t *parser, return 0; return yaml_parser_parse_node(parser, event, 0, 0); } + else if (token->type == YAML_FLOW_SEQUENCE_END_TOKEN) { + yaml_mark_t mark = token->start_mark; + parser->state = YAML_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE; + return yaml_parser_process_empty_scalar(parser, event, mark); + } else { yaml_mark_t mark = token->end_mark; SKIP_TOKEN(parser); diff --git a/src/YamlCppLib/libyaml/src/scanner.c b/src/YamlCppLib/libyaml/src/scanner.c @@ -273,7 +273,7 @@ * The tokens BLOCK-SEQUENCE-START and BLOCK-MAPPING-START denote indentation * increase that precedes a block collection (cf. the INDENT token in Python). * The token BLOCK-END denote indentation decrease that ends a block collection - * (cf. the DEDENT token in Python). However YAML has some syntax pecularities + * (cf. the DEDENT token in Python). However YAML has some syntax peculiarities * that makes detections of these tokens more complex. * * The tokens BLOCK-ENTRY, KEY, and VALUE are used to represent the indicators @@ -3287,7 +3287,7 @@ yaml_parser_scan_flow_scalar(yaml_parser_t *parser, yaml_token_t *token, /* Check if we are at the end of the scalar. */ - /* Fix for crash unitialized value crash + /* Fix for crash uninitialized value crash * Credit for the bug and input is to OSS Fuzz * Credit for the fix to Alex Gaynor */ diff --git a/src/YamlCppLib/libyaml/tests/run-emitter-test-suite.c b/src/YamlCppLib/libyaml/tests/run-emitter-test-suite.c @@ -72,7 +72,7 @@ int main(int argc, char *argv[]) assert(input); if (!yaml_emitter_initialize(&emitter)) { - fprintf(stderr, "Could not initalize the emitter object\n"); + fprintf(stderr, "Could not initialize the emitter object\n"); return 1; } yaml_emitter_set_output_file(&emitter, stdout); diff --git a/src/YamlCppLib/libyaml/tests/run-parser-test-suite.c b/src/YamlCppLib/libyaml/tests/run-parser-test-suite.c @@ -14,9 +14,16 @@ int main(int argc, char *argv[]) int flow = -1; /** default no flow style collections */ int i = 0; int foundfile = 0; + int max_level; + char *output; for (i = 1; i < argc; i++) { - if (strncmp(argv[i], "--flow", 6) == 0) { + if (strncmp(argv[i], "--max-level", 11) == 0) { + i++; + max_level = strtol(argv[i], &output, 10); + yaml_set_max_nest_level(max_level); + } + else if (strncmp(argv[i], "--flow", 6) == 0) { if (i+1 == argc) return usage(1); i++; diff --git a/src/YamlCppLib/libyaml/tests/run-parser.c b/src/YamlCppLib/libyaml/tests/run-parser.c @@ -12,13 +12,31 @@ int main(int argc, char *argv[]) { int number; + int start = 0; + int i = 0; + char *filename; + char *output; + int max_level; + int show_error = 0; if (argc < 2) { printf("Usage: %s file1.yaml ...\n", argv[0]); return 0; } + for (i = 1; i < argc; i++) { + if (strncmp(argv[i], "--max-level", 11) == 0) { + i++; + max_level = strtol(argv[i], &output, 10); + yaml_set_max_nest_level(max_level); + start = i+1; + } + else if (strncmp(argv[i], "--show-error", 12) == 0) { + show_error = 1; + start = i+1; + } + } - for (number = 1; number < argc; number ++) + for (number = start; number < argc; number ++) { FILE *file; yaml_parser_t parser; @@ -27,10 +45,11 @@ main(int argc, char *argv[]) int count = 0; int error = 0; - printf("[%d] Parsing '%s': ", number, argv[number]); + filename = argv[number]; + printf("[%d] Parsing '%s': ", number, filename); fflush(stdout); - file = fopen(argv[number], "rb"); + file = fopen(filename, "rb"); assert(file); assert(yaml_parser_initialize(&parser)); @@ -41,6 +60,12 @@ main(int argc, char *argv[]) { if (!yaml_parser_parse(&parser, &event)) { error = 1; + if (show_error) { + fprintf(stderr, "Parse error: %s\nLine: %lu Column: %lu\n", + parser.problem, + (unsigned long)parser.problem_mark.line + 1, + (unsigned long)parser.problem_mark.column + 1); + } break; } diff --git a/src/YamlCppLib/libyaml/tests/test-reader.c b/src/YamlCppLib/libyaml/tests/test-reader.c @@ -103,7 +103,7 @@ test_case utf8_sequences[] = { test_case boms[] = { - /* {"title", "test!", lenth}, */ + /* {"title", "test!", length}, */ {"no bom (utf-8)", "Hi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82!", 13}, {"bom (utf-8)", "\xef\xbb\xbfHi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82!", 13}, diff --git a/src/YamlCppLib/readme.md b/src/YamlCppLib/readme.md @@ -1,6 +1,6 @@ ## YamlCppLib -Do not change code under the libyaml directory; it contains libyaml source code version [0.2.5](https://github.com/yaml/libyaml/releases/tag/0.2.5). +Do not change code under the libyaml directory; it contains libyaml source code version [840b65c](https://github.com/yaml/libyaml/commits/840b65c). It is created using git subtree command: git subtree add --prefix=src/YamlCppLib/libyaml https://github.com/yaml/libyaml.git release/0.2.5 --squash