diff --git a/.travis.yml b/.travis.yml index 9b30190c..0181435c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ python: - 3.6 - 3.7 - 3.8 - # - 3.9 pylint will not support Python 3.9 until https://github.com/PyCQA/pylint/issues/3882 is resolved + - 3.9 os: linux dist: xenial diff --git a/CHANGES b/CHANGES index ec70b5a8..bb4d5bb2 100644 --- a/CHANGES +++ b/CHANGES @@ -7,14 +7,47 @@ Bug Fixes: for appropriate null/None leaf nodes. Enhancements: +* Python 3.9 is now supported (because common testing tools finally work with + Python 3.9). * The node deletion capability of the yaml-set command is now part of the library. See Processor::delete_nodes(...) and Processor::delete_gathered_nodes(...) for details. +* The node aliasing capability of the yaml-set command is now part of the + library. See Processor::alias_nodes(...) and + Processor::alias_gathered_nodes(...) for details. +* The node tagging capability of the yaml-set command is now part of the + library. See Processor::tag_nodes(...) and + Processor::tag_gathered_nodes(...) for details. * The library now supports loading YAML from String rather than only from file. Simply pass a new `literal=True` keyword parameter to Parsers::get_yaml_data(...) or Parsers::get_yaml_multidoc_data(...) to indicate that `source` is literal serialized (String) YAML data rather than a file-spec. This mode is implied when reading from STDIN (source is "-"). +* The emitter_write_folded_fix.py patch file for ruamel.yaml has been removed + in favor of an author-supplied solution to the problem -- + https://sourceforge.net/p/ruamel-yaml/tickets/383/ -- for which the patch was + originally written. + +Known Issues: +* ruamel.yaml version 0.17.x is a major refactoring effort by the project's + owner. As such, only select versions will be marked as compatible with + yamlpath. Such marking occurs in this project's dependencies list via the + setup.py file. This is necessary because I use yamlpath in production + environments where stability is paramount; I need the freedom to update + yamlpath at-will without incurring any unexpected failures due to + incompatible ruamel.yaml changes. I will try to test some -- but not all -- + ruamel.yaml releases from time to time and update yamlpath dependency + compatibilities accordingly. +* ruamel.yaml version 0.17.4 somewhat resolves a previously reported issue -- + https://sourceforge.net/p/ruamel-yaml/tickets/351/ -- wherein certain + arrangements of comments or new-lines within YAML files near aliased hash + keys would cause a total loss of data when the stream was written to file. + Now, the data is no longer entirely lost. However, the preceding comment or + new-line is deleted when the stream is written to file. This is deemed to be + an acceptable compromise, for now, because the alternative is to either lose + the entire document or lose all attempted changes to the affected document. + Until the issue is properly fixed, an XFAIL test will continue to be in the + yamlpath unit test suite. 3.4.0: Bug Fixes: diff --git a/README.md b/README.md index 665fcffb..18e5866c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # YAML Path and Command-Line Tools -[![Build Status](https://travis-ci.org/wwkimball/yamlpath.svg?branch=master)](https://travis-ci.org/wwkimball/yamlpath) +[![Build Status](https://api.travis-ci.com/wwkimball/yamlpath.svg?branch=master)](https://travis-ci.com/github/wwkimball/yamlpath) [![Python versions](https://img.shields.io/pypi/pyversions/yamlpath.svg)](https://pypi.org/project/yamlpath/) [![PyPI version](https://badge.fury.io/py/yamlpath.svg)](https://pypi.org/project/yamlpath/) [![Downloads](https://pepy.tech/badge/yamlpath)](https://pepy.tech/project/yamlpath) @@ -21,9 +21,13 @@ for other projects to readily employ YAML Paths. 1. [Introduction](#introduction) 2. [Illustration](#illustration) -3. [Installing](#installing) -4. [Supported YAML Path Segments](#supported-yaml-path-segments) -5. [Based on ruamel.yaml and Python 3](#based-on-ruamelyaml-and-python-3) +3. [Supported YAML Path Segments](#supported-yaml-path-segments) +4. [Installing](#installing) + 1. [Requirements](#requirements) + 2. [Using pip](#using-pip) + 1. [Very Old Versions of pip or its setuptools Dependency](#very-old-versions-of-pip-or-its-setuptools-dependency) + 3. [Installing EYAML (Optional)](#installing-eyaml-optional) +5. [Based on ruamel.yaml](#based-on-ruamelyaml) 6. [The Files of This Project](#the-files-of-this-project) 1. [Command-Line Tools](#command-line-tools) 2. [Libraries](#libraries) @@ -239,35 +243,62 @@ The [project Wiki provides more illustrative details of YAML Path Segments](http ## Installing -This project requires [Python](https://www.python.org/) 3. It is tested -against Pythons 3.6 through 3.8. Most operating systems and distributions -have access to Python 3 even if only Python 2 -- or no Python, at all -- came -pre-installed. It is generally safe to have more than one version of Python -on your system at the same time, especially when using +Some OS distributions offer some versions of yamlpath -- and its dependencies +-- via packages. While these versions of yamlpath are often outdated, they can +be convenient to install using your OS' native package manager (`apt`, `yum`, +`npm`, and such). Otherwise, Python's own package manager `pip` will always +offer the latest version of yamlpath and -- even better -- can be isolated to +ephemeral or longer-lasting virtual Python environments. + +### Requirements + +This project requires [Python](https://www.python.org/) 3. It is rigorously +tested against Pythons 3.6 through 3.9. Most operating systems and +distributions have access to Python 3 even if only Python 2 -- or no Python, at +all -- came pre-installed. It is generally safe to have more than one version +of Python on your system at the same time, especially when using [virtual Python environments](https://docs.python.org/3/library/venv.html). -Each published version of this project can be installed from -[PyPI](https://pypi.org/) using `pip`. Note that on systems with more than one -version of Python, you will probably need to use `pip3`, or equivalent (e.g.: -Cygwin users may need to use `pip3.6`, `pip3.7`, `pip3.8`, or such). +*yamlpath* depends on *ruamel.yaml* (derived from and greatly extending PyYAML). +When using OS-native packages or `pip`, you do not need to pre-install +*ruamel.yaml* except under extraordinary circumstances like using very old +versions of `pip` or its own dependency, *setuptools*. + +### Using pip + +Each published version of this project and its dependencies can be installed +from [PyPI](https://pypi.org/) using `pip`. Note that on systems with more than +one version of Python, you will probably need to use `pip3`, or equivalent +(e.g.: Cygwin users may need to use `pip3.6`, `pip3.9`, or such). ```shell pip3 install yamlpath ``` -Note that very old versions of Python 3 ship with seriously outdated versions -of pip and setuptools. You *must* update to at least **pip** version **18.1** -and **setuptools** version **46.4.0** to install yamlpath without -pre-installing its dependencies. If you cannot update pip or setuptools, you -can still install yamlpath except you'll first need to install **ruamel.yaml** -like so: +#### Very Old Versions of pip or its setuptools Dependency + +Very old versions of Python 3 ship with seriously outdated versions of `pip` and +its *setuptools* dependency. When using versions of `pip` older than **18.1** +or *setuptools* older than version **46.4.0**, you may not be able to install +yamlpath with a single command. In this case, you have two options: either +pre-install *ruamel.yaml* before installing *yamlpath* or update `pip` and/or +*setuptools* to at least the minimum required versions so `pip` can +auto-determine and install dependencies. This issue is not unique to yamlpath +because Python's ever-growing capabilities simply require periodic updates to +access. + +When you cannot update `pip` or *setuptools*, just pre-install *ruamel.yaml* +before yamlpath, like so: ```shell -# These commands CANNOT be joined, like: pip3.6 install ruamel.yaml yamlpath +# In this edge-case, these commands CANNOT be joined, like: +# pip3.6 install ruamel.yaml yamlpath pip3.6 install ruamel.yaml pip3.6 install yamlpath ``` +### Installing EYAML (Optional) + EYAML support is entirely optional. You do not need EYAML to use YAML Path. That YAML Path supports EYAML is a service to a substantial audience: Puppet users. At the time of this writing, EYAML (classified as a Hiera @@ -284,7 +315,7 @@ If this puts the `eyaml` command on your system `PATH`, nothing more need be done apart from generating or obtaining your encryption keys. Otherwise, you can tell YAML Path library and tools where to find the `eyaml` command. -## Based on ruamel.yaml and Python 3 +## Based on ruamel.yaml In order to support the best available YAML editing capability (so called, round-trip editing with support for comment preservation), this project is based @@ -659,18 +690,19 @@ https://github.com/wwkimball/yamlpath. ```text usage: yaml-set [-h] [-V] -g YAML_PATH - [-a VALUE | -f FILE | -i | -R LENGTH | -D] + [-a VALUE | -A ANCHOR | -f FILE | -i | -R LENGTH | -N | -D] [-F {bare,boolean,default,dquote,float,folded,int,literal,squote}] [-c CHECK] [-s YAML_PATH] [-m] [-b] - [-t ['.', '/', 'auto', 'dot', 'fslash']] [-M CHARS] [-e] - [-x EYAML] [-r PRIVATEKEY] [-u PUBLICKEY] [-S] [-d | -v | -q] + [-t ['.', '/', 'auto', 'dot', 'fslash']] [-M CHARS] [-H ANCHOR] + [-T TAG] [-e] [-x EYAML] [-r PRIVATEKEY] [-u PUBLICKEY] [-S] + [-d | -v | -q] [YAML_FILE] Changes one or more Scalar values in a YAML/JSON/Compatible document at a specified YAML Path. Matched values can be checked before they are replaced to mitigate accidental change. When matching singular results, the value can be -archived to another key before it is replaced. Further, EYAML can be employed -to encrypt the new values and/or decrypt an old value before checking it. +archived to another key before it is replaced. Further, EYAML can be employed to +encrypt the new values and/or decrypt an old value before checking it. positional arguments: YAML_FILE the YAML file to update; omit or use - to read from @@ -691,14 +723,20 @@ optional arguments: -b, --backup save a backup YAML_FILE with an extra .bak file- extension -t ['.', '/', 'auto', 'dot', 'fslash'], --pathsep ['.', '/', 'auto', 'dot', 'fslash'] - indicate which YAML Path seperator to use when - rendering results; default=dot + indicate which YAML Path seperator to use when rendering + results; default=dot -M CHARS, --random-from CHARS characters from which to build a value for --random; - default=all upper- and lower-case letters and all - digits - -S, --nostdin Do not implicitly read from STDIN, even when there is - no YAML_FILE with a non-TTY session + default=all upper- and lower-case letters and all digits + -H ANCHOR, --anchor ANCHOR + when --aliasof|-A points to a value which is not already + Anchored, a new Anchor with this name is created; + renames an existing Anchor if already set + -T TAG, --tag TAG assign a custom YAML (data-type) tag to the changed + nodes; can be used without other input options to assign + or change a tag + -S, --nostdin Do not implicitly read from STDIN, even when there is no + YAML_FILE with a non-TTY session -d, --debug output debugging details -v, --verbose increase output verbosity -q, --quiet suppress all output except errors @@ -709,20 +747,24 @@ required settings: input options: -a VALUE, --value VALUE - set the new value from the command-line instead of - STDIN + set the new value from the command-line instead of STDIN + -A ANCHOR, --aliasof ANCHOR + set the value as a YAML Alias of an existing Anchor, by + name (merely copies the target value for non-YAML files) -f FILE, --file FILE read the new value from file (discarding any trailing new-lines) -i, --stdin accept the new value from STDIN (best for sensitive data) -R LENGTH, --random LENGTH randomly generate a replacement value of a set length - -D, --delete delete rather than change target node(s) + -N, --null sets the value to null + -D, --delete delete rather than change target node(s); implies + --mustexist|-m EYAML options: - Left unset, the EYAML keys will default to your system or user defaults. - You do not need to supply a private key unless you enable --check and the - old value is encrypted. + Left unset, the EYAML keys will default to your system or user defaults. You + do not need to supply a private key unless you enable --check and the old + value is encrypted. -e, --eyamlcrypt encrypt the new value using EYAML -x EYAML, --eyaml EYAML @@ -734,7 +776,9 @@ EYAML options: When no changes are made, no backup is created, even when -b/--backup is specified. For more information about YAML Paths, please visit -https://github.com/wwkimball/yamlpath. +https://github.com/wwkimball/yamlpath/wiki. To report issues with this tool or +to request enhancements, please visit +https://github.com/wwkimball/yamlpath/issues. ``` * [yaml-validate](yamlpath/commands/yaml_validate.py) @@ -773,7 +817,7 @@ exceptions, the most interesting library files include: or write data to YAML/Compatible sources. * [eyamlprocessor.py](yamlpath/eyaml/eyamlprocessor.py) -- Extends the Processor class to support EYAML data encryption and decryption. -* [merger.py](merger/merger.py) -- The core document merging logic. +* [merger.py](yamlpath/merger/merger.py) -- The core document merging logic. ## Basic Usage @@ -1067,14 +1111,6 @@ and editing with ruamel.yaml. When you need to process EYAML encrypted data, replace `yamlpath.Processor` with `yamlpath.eyaml.EYAMLProcessor` and add error handling for `yamlpath.eyaml.EYAMLCommandException`. -Note that `import yamlpath.patches` is entirely optional. I wrote and use it to -block ruamel.yaml's Emitter from injecting unnecessary newlines into folded -values (it improperly converts every single new-line into two for left-flushed -multi-line values, at the time of this writing). Since "block" output EYAML -values are left-flushed multi-line folded strings, this fix is necessary when -using EYAML features. At least, until ruamel.yaml has its own fix for this -issue. - Note also that these examples use `ConsolePrinter` to handle STDOUT and STDERR messaging. You don't have to. However, some kind of logger must be passed to these libraries so they can write messages _somewhere_. Your custom message @@ -1090,7 +1126,6 @@ import sys from ruamel.yaml import YAML from ruamel.yaml.parser import ParserError -import yamlpath.patches from yamlpath.func import get_yaml_data, get_yaml_editor from yamlpath.wrappers import ConsolePrinter from yamlpath import Processor diff --git a/setup.py b/setup.py index 2bc12798..9415f315 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Operating System :: OS Independent", "Topic :: Software Development :: Libraries :: Python Modules", ], @@ -40,7 +41,7 @@ }, python_requires=">3.6.0", install_requires=[ - "ruamel.yaml>=0.15.96", + "ruamel.yaml>=0.15.96,!=0.17.0,!=0.17.1,!=0.17.2,<=0.17.4", ], tests_require=[ "pytest", diff --git a/tests/test_eyaml_eyamlprocessor.py b/tests/test_eyaml_eyamlprocessor.py index 855236a2..cdb898bf 100644 --- a/tests/test_eyaml_eyamlprocessor.py +++ b/tests/test_eyaml_eyamlprocessor.py @@ -4,7 +4,6 @@ from ruamel.yaml import YAML -import yamlpath.patches from yamlpath.func import unwrap_node_coords from yamlpath.enums import YAMLValueFormats from yamlpath.eyaml.enums import EYAMLOutputFormats diff --git a/tests/test_processor.py b/tests/test_processor.py index 55983100..2607c73d 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -3,6 +3,7 @@ from types import SimpleNamespace from ruamel.yaml import YAML +from ruamel.yaml.comments import TaggedScalar from yamlpath.func import unwrap_node_coords from yamlpath.exceptions import YAMLPathException @@ -830,3 +831,82 @@ def test_null_docs_have_nothing_to_delete(self, capsys): console = capsys.readouterr() assert "Refusing to delete nodes from a null document" in console.out + + def test_null_docs_have_nothing_to_gather_and_alias(self, capsys): + args = SimpleNamespace(verbose=False, quiet=False, debug=True) + logger = ConsolePrinter(args) + processor = Processor(logger, None) + + processor.alias_nodes("/alias*", "/anchor") + + console = capsys.readouterr() + assert "Refusing to alias nodes in a null document" in console.out + + def test_null_docs_have_nothing_to_alias(self, capsys): + args = SimpleNamespace(verbose=False, quiet=False, debug=True) + logger = ConsolePrinter(args) + processor = Processor(logger, None) + + processor.alias_gathered_nodes([], "/anchor") + + console = capsys.readouterr() + assert "Refusing to alias nodes in a null document" in console.out + + def test_null_docs_have_nothing_to_tag(self, capsys): + args = SimpleNamespace(verbose=False, quiet=False, debug=True) + logger = ConsolePrinter(args) + processor = Processor(logger, None) + + processor.tag_nodes("/tag_nothing", "tag_this") + + console = capsys.readouterr() + assert "Refusing to tag nodes from a null document" in console.out + + @pytest.mark.parametrize("alias_path,anchor_path,anchor_name,pathseperator", [ + (YAMLPath("/a_hash/a_key"), YAMLPath("/some_key"), "", PathSeperators.FSLASH), + ("a_hash.a_key", "some_key", "", PathSeperators.AUTO), + ]) + def test_anchor_nodes(self, quiet_logger, alias_path, anchor_path, anchor_name, pathseperator): + anchor_value = "This is the Anchored value!" + yamlin = """--- +some_key: {} +a_hash: + a_key: A value +""".format(anchor_value) + + yaml = YAML() + data = yaml.load(yamlin) + processor = Processor(quiet_logger, data) + + processor.alias_nodes( + alias_path, anchor_path, + pathsep=pathseperator, anchor_name=anchor_name) + + match_count = 0 + for node in processor.get_nodes( + alias_path, mustexist=True + ): + match_count += 1 + assert unwrap_node_coords(node) == anchor_value + assert match_count == 1 + + @pytest.mark.parametrize("yaml_path,tag,pathseperator", [ + (YAMLPath("/key"), "!taggidy", PathSeperators.FSLASH), + ("key", "taggidy", PathSeperators.AUTO), + ]) + def test_tag_nodes(self, quiet_logger, yaml_path, tag, pathseperator): + yamlin = """--- +key: value +""" + + yaml = YAML() + data = yaml.load(yamlin) + processor = Processor(quiet_logger, data) + + processor.tag_nodes(yaml_path, tag, pathsep=pathseperator) + + if tag and not tag[0] == "!": + tag = "!{}".format(tag) + + assert isinstance(data['key'], TaggedScalar) + assert data['key'].tag.value == tag diff --git a/tests/test_wrappers_consoleprinter.py b/tests/test_wrappers_consoleprinter.py index 0fea10d0..e6590ffc 100644 --- a/tests/test_wrappers_consoleprinter.py +++ b/tests/test_wrappers_consoleprinter.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from ruamel.yaml.comments import CommentedMap, CommentedSeq, TaggedScalar -from ruamel.yaml.scalarstring import PlainScalarString +from ruamel.yaml.scalarstring import PlainScalarString, FoldedScalarString from yamlpath.wrappers import NodeCoords from yamlpath.wrappers import ConsolePrinter @@ -57,6 +57,10 @@ def test_debug_noisy(self, capsys): logger = ConsolePrinter(args) anchoredkey = PlainScalarString("TestKey", anchor="KeyAnchor") anchoredval = PlainScalarString("TestVal", anchor="Anchor") + foldedstr = "123456789 123456789 123456789" + foldedstrfolds = [10, 20] + foldedval = FoldedScalarString(foldedstr) + foldedval.fold_pos = foldedstrfolds logger.debug(anchoredval) console = capsys.readouterr() @@ -170,6 +174,12 @@ def test_debug_noisy(self, capsys): "DEBUG: test_debug_noisy: (parentref)key", ]) + "\n" == console.out + logger.debug(foldedval) + console = capsys.readouterr() + assert "\n".join([ + "DEBUG: {},folded@{}".format(foldedstr, foldedstrfolds) + ]) + def test_debug_quiet(self, capsys): args = SimpleNamespace(verbose=False, quiet=True, debug=True) logger = ConsolePrinter(args) diff --git a/yamlpath/commands/eyaml_rotate_keys.py b/yamlpath/commands/eyaml_rotate_keys.py index 7561aa3a..441ed154 100644 --- a/yamlpath/commands/eyaml_rotate_keys.py +++ b/yamlpath/commands/eyaml_rotate_keys.py @@ -18,9 +18,6 @@ from yamlpath.common import Parsers from yamlpath.eyaml.exceptions import EYAMLCommandException from yamlpath.eyaml import EYAMLProcessor - -# pylint: disable=locally-disabled,unused-import -import yamlpath.patches from yamlpath.wrappers import ConsolePrinter def processcli(): diff --git a/yamlpath/commands/yaml_set.py b/yamlpath/commands/yaml_set.py index c13ac71a..b9c80e04 100644 --- a/yamlpath/commands/yaml_set.py +++ b/yamlpath/commands/yaml_set.py @@ -19,20 +19,16 @@ from os.path import isfile, exists from shutil import copy2, copyfileobj from pathlib import Path -from typing import Any, Dict from yamlpath import __version__ as YAMLPATH_VERSION -from yamlpath.common import Anchors, Nodes, Parsers +from yamlpath.common import Nodes, Parsers from yamlpath import YAMLPath from yamlpath.exceptions import YAMLPathException from yamlpath.enums import YAMLValueFormats, PathSeperators from yamlpath.eyaml.exceptions import EYAMLCommandException from yamlpath.eyaml.enums import EYAMLOutputFormats from yamlpath.eyaml import EYAMLProcessor - -# pylint: disable=locally-disabled,unused-import -import yamlpath.patches -from yamlpath.wrappers import ConsolePrinter, NodeCoords +from yamlpath.wrappers import ConsolePrinter def processcli(): """Process command-line arguments.""" @@ -301,7 +297,11 @@ def save_to_yaml_file(args, log, yaml_parser, yaml_data, backup_file): with open(args.yaml_file, 'w') as yaml_dump: try: yaml_parser.dump(yaml_data, yaml_dump) - except AssertionError as ex: + # Tell pycov to ignore this block because it is impossible to + # trigger it for ruamel.yaml versions >0.17.4 yet this project must + # continue to support older versions of ruamel.yaml as long as OS + # package builders continue to be dependent on them. + except AssertionError as ex: # pragma: no cover yaml_dump.close() tmphnd.seek(0) with open(args.yaml_file, 'wb') as outhnd: @@ -414,61 +414,11 @@ def _alias_nodes( log, processor, assign_to_nodes, anchor_path, anchor_name ): """Assign YAML Aliases to the target nodes.""" - anchor_node_coordinates = _get_nodes( - log, processor, anchor_path, must_exist=True) - if len(anchor_node_coordinates) > 1: - log.critical( - "It is impossible to Alias more than one Anchor at a time from {}!" - .format(anchor_path), 1) - - anchor_coord = anchor_node_coordinates[0] - anchor_node = anchor_coord.node - if not hasattr(anchor_node, "anchor"): - anchor_coord.parent[anchor_coord.parentref] = Nodes.wrap_type( - anchor_node) - anchor_node = anchor_coord.parent[anchor_coord.parentref] - - known_anchors: Dict[str, Any] = {} - Anchors.scan_for_anchors(processor.data, known_anchors) - - if anchor_name: - # Rename any pre-existing anchor or set an original anchor name; the - # assigned name must be unique! - if anchor_name in known_anchors: - log.critical( - "Anchor names must be unique within YAML documents. Anchor" - " name, {}, is already used.".format(anchor_name)) - anchor_node.yaml_set_anchor(anchor_name, always_dump=True) - elif anchor_node.anchor.value: - # The source node already has an anchor name - anchor_name = anchor_node.anchor.value - else: - # An orignial, unique-to-the-document anchor name must be generated - new_anchor = Anchors.generate_unique_anchor_name( - processor.data, anchor_coord, known_anchors) - anchor_node.yaml_set_anchor(new_anchor, always_dump=True) - - for node_coord in assign_to_nodes: - log.debug( - "Attempting to set the anchor name for node to {}:" - .format(anchor_name), - data=node_coord, - prefix="yaml_set::_alias_nodes: ") - node_coord.parent[node_coord.parentref] = anchor_node - -def _tag_nodes(document, tag, nodes): - """Assign a data-type tag to a set of nodes.""" - for node_coord in nodes: - old_node = node_coord.node - if node_coord.parent is None: - node_coord.node.yaml_set_tag(tag) - else: - node_coord.parent[node_coord.parentref] = Nodes.apply_yaml_tag( - node_coord.node, tag) - if Anchors.get_node_anchor(old_node) is not None: - Anchors.replace_anchor( - document, old_node, - node_coord.parent[node_coord.parentref]) + try: + processor.alias_gathered_nodes( + assign_to_nodes, anchor_path, anchor_name=anchor_name) + except YAMLPathException as ex: + log.critical(ex, 1) # pylint: disable=locally-disabled,too-many-locals,too-many-branches,too-many-statements def main(): @@ -641,7 +591,7 @@ def main(): except YAMLPathException as ex: log.critical(ex, 1) elif args.tag: - _tag_nodes(processor.data, args.tag, change_node_coordinates) + processor.tag_gathered_nodes(change_node_coordinates, args.tag) # Write out the result write_output_document(args, log, yaml, yaml_data) diff --git a/yamlpath/common/nodes.py b/yamlpath/common/nodes.py index 9d62490f..637283ef 100644 --- a/yamlpath/common/nodes.py +++ b/yamlpath/common/nodes.py @@ -88,6 +88,26 @@ def make_new_node( elif valform == YAMLValueFormats.FOLDED: new_type = FoldedScalarString new_value = str(value) + preserve_folds = [] + + # Scan all except the very last character because if that last + # character is a newline, ruamel.yaml will crash when told to fold + # there. + for index, fold_char in enumerate(new_value[:-1]): + if fold_char == "\n": + preserve_folds.append(index) + + # Replace all except the very last character + new_value = new_value[:-1].replace("\n", " ") + new_value[-1] + + if hasattr(source_node, "anchor") and source_node.anchor.value: + new_node = new_type(new_value, anchor=source_node.anchor.value) + else: + new_node = new_type(new_value) + + if preserve_folds: + new_node.fold_pos = preserve_folds # type: ignore + elif valform == YAMLValueFormats.LITERAL: new_type = LiteralScalarString new_value = str(value) diff --git a/yamlpath/patches/__init__.py b/yamlpath/patches/__init__.py deleted file mode 100644 index 83cac749..00000000 --- a/yamlpath/patches/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Make all patches available.""" -import yamlpath.patches.emitter_write_folded_fix diff --git a/yamlpath/patches/emitter_write_folded_fix.py b/yamlpath/patches/emitter_write_folded_fix.py deleted file mode 100644 index d9c7f579..00000000 --- a/yamlpath/patches/emitter_write_folded_fix.py +++ /dev/null @@ -1,94 +0,0 @@ -""" -Patch bugs in ruamel.yaml. - -This will exist unless or until they are patched in the ruamel.yaml package -itself. - -Copyright 2018, 2019, 2020 William W. Kimball, Jr. MBA MSIS -""" -from typing import Any -from ruamel.yaml.emitter import ( - Emitter, - EmitterError, -) - - -# Stop Emitter.write_folded from injecting unnecessary new-lines -def write_folded_fix(self, text): - # type: (Emitter, Any) -> None - """Make pep257 happy...""" - hints, _indent, _indicator = self.determine_block_hints(text) - self.write_indicator(u'>' + hints, True) - if _indicator == u'+': - self.open_ended = True - self.write_line_break() - leading_space = True - spaces = False - breaks = True - start = end = 0 - while end <= len(text): - ch = None - if end < len(text): - ch = text[end] - if breaks: - if ch is None or ch not in u'\n\x85\u2028\u2029\a': - if ( - not leading_space - and ch is not None - and ch != u' ' - and text[start] == u'\n' - ): - self.write_line_break() - leading_space = ch == u' ' - - # This must apply only to the very last character - if end == len(text): - for br in text[start:end]: - if br == u'\n': - self.write_line_break() - else: - self.write_line_break(br) - - if ch is not None: - self.write_indent() - start = end - elif spaces: - if ch != u' ': - if start + 1 == end and self.column > self.best_width: - self.write_indent() - else: - data = text[start:end] - self.column += len(data) - if bool(self.encoding): - data = data.encode(self.encoding) - self.stream.write(data) - start = end - else: - if ch is None or ch in u' \n\x85\u2028\u2029\a': - data = text[start:end] - self.column += len(data) - if bool(self.encoding): - data = data.encode(self.encoding) - self.stream.write(data) - if ch == u'\a': - if end < (len(text) - 1) and not text[end + 2].isspace(): - self.write_line_break() - self.write_indent() - # \a and the space that is inserted on the fold - end += 2 - else: - raise EmitterError( - 'unexcpected fold indicator \\a before space' - ) - if ch is None: - self.write_line_break() - start = end - if ch is not None: - breaks = ch in u'\n\x85\u2028\u2029' - spaces = ch == u' ' - end += 1 - - -# MYPY hates MonkeyPatching per https://github.com/python/mypy/issues/2427 -# but there's no choice here, so... ignore the type. -Emitter.write_folded = write_folded_fix # type: ignore diff --git a/yamlpath/processor.py b/yamlpath/processor.py index c0f3db06..8c888e9a 100644 --- a/yamlpath/processor.py +++ b/yamlpath/processor.py @@ -4,9 +4,9 @@ Copyright 2018, 2019, 2020 William W. Kimball, Jr. MBA MSIS """ -from typing import Any, Generator, List, Union +from typing import Any, Dict, Generator, List, Union -from yamlpath.common import Nodes, Searches +from yamlpath.common import Anchors, Nodes, Searches from yamlpath import YAMLPath from yamlpath.path import SearchTerms, CollectorTerms from yamlpath.wrappers import ConsolePrinter, NodeCoords @@ -191,10 +191,246 @@ def set_value(self, yaml_path: Union[YAMLPath, str], .format(value, value_format, str(vex)) , str(yaml_path)) from vex + def _get_anchor_node( + self, anchor_path: Union[YAMLPath, str], **kwargs: Any + ) -> Any: + """ + Gather the source YAML Anchor node for an Aliasing operation. + + Parameters: + 1. anchor_path (Union[YAMLPath, str]) The YAML Path to a single source + anchor node; specifying any path which points to more than one node + will result in a YAMLPathException because YAML does not define + Aliases for more than one Anchor. + + Keyword Parameters: + * anchor_name (str) Alternate name to use for the YAML Anchor and its + Aliases. + + Returns: (Any) The source node + + Raises: + - `YAMLPathException` when YAML Path is invalid or a supplied + anchor_name is illegal + """ + pathsep: PathSeperators = kwargs.pop("pathsep", PathSeperators.AUTO) + anchor_name: str = kwargs.pop("anchor_name", "") + + if isinstance(anchor_path, str): + anchor_path = YAMLPath(anchor_path, pathsep) + elif pathsep is not PathSeperators.AUTO: + anchor_path.seperator = pathsep + + anchor_node_coordinates: List[NodeCoords] = [] + for node_coords in self._get_required_nodes(self.data, anchor_path): + self.logger.debug( + "Gathered YAML Anchor node:", + prefix="Processor::_get_anchor_node: ", data=node_coords) + anchor_node_coordinates.append(node_coords) + if len(anchor_node_coordinates) > 1: + raise YAMLPathException( + "It is impossible to Alias more than one Anchor at a time!", + str(anchor_path)) + + anchor_coord = anchor_node_coordinates[0] + anchor_node = anchor_coord.node + if not hasattr(anchor_node, "anchor"): + anchor_coord.parent[anchor_coord.parentref] = Nodes.wrap_type( + anchor_node) + anchor_node = anchor_coord.parent[anchor_coord.parentref] + + known_anchors: Dict[str, Any] = {} + Anchors.scan_for_anchors(self.data, known_anchors) + + if anchor_name: + # Rename any pre-existing anchor or set an original anchor name; + # the assigned name must be unique! + if anchor_name in known_anchors: + raise YAMLPathException( + "Anchor names must be unique within YAML documents." + " Anchor name, {}, is already used." + .format(anchor_name), str(anchor_path)) + anchor_node.yaml_set_anchor(anchor_name, always_dump=True) + elif anchor_node.anchor.value: + # The source node already has an anchor name + anchor_name = anchor_node.anchor.value + else: + # An orignial, unique-to-the-document anchor name must be generated + new_anchor = Anchors.generate_unique_anchor_name( + self.data, anchor_coord, known_anchors) + anchor_node.yaml_set_anchor(new_anchor, always_dump=True) + + return anchor_node + + def alias_nodes( + self, yaml_path: Union[YAMLPath, str], + anchor_path: Union[YAMLPath, str], **kwargs: Any + ) -> None: + """ + Gather and assign YAML Aliases to nodes at YAML Path in data. + + Parameters: + 1. yaml_path (Union[YAMLPath, str]) The YAML Path to all target nodes + which will become Aliases to the Anchor node specified via + `anchor_path`. + 2. anchor_path (Union[YAMLPath, str]) The YAML Path to a single source + anchor node; specifying any path which points to more than one node + will result in a YAMLPathException because YAML does not define + Aliases for more than one Anchor. + + Keyword Parameters: + * pathsep (PathSeperators) Forced YAML Path segment seperator; set + only when automatic inference fails; + default = PathSeperators.AUTO + * anchor_name (str) Override the Alias name to any non-empty name you + set; attempts to re-use an existing Anchor name will result in a + YAMLPathException. + + Returns: N/A + + Raises: + - `YAMLPathException` when YAML Path is invalid + """ + pathsep: PathSeperators = kwargs.pop("pathsep", PathSeperators.AUTO) + anchor_name: str = kwargs.pop("anchor_name", "") + + if self.data is None: + self.logger.debug( + "Refusing to alias nodes in a null document!", + prefix="Processor::alias_nodes: ", data=self.data) + return + + if isinstance(yaml_path, str): + yaml_path = YAMLPath(yaml_path, pathsep) + elif pathsep is not PathSeperators.AUTO: + yaml_path.seperator = pathsep + + anchor_node = self._get_anchor_node( + anchor_path, pathsep=pathsep, anchor_name=anchor_name) + + gathered_nodes: List[NodeCoords] = [] + for node_coords in self._get_required_nodes(self.data, yaml_path): + self.logger.debug( + "Gathered node for YAML Alias assignment:", + prefix="Processor::delete_nodes: ", data=node_coords) + gathered_nodes.append(node_coords) + + if len(gathered_nodes) > 0: + self._alias_nodes(gathered_nodes, anchor_node) + + def alias_gathered_nodes( + self, gathered_nodes: List[NodeCoords], + anchor_path: Union[YAMLPath, str], **kwargs: Any + ) -> None: + """ + Assign a YAML Anchor to zero or more YAML Alias nodes. + + Parameters: + 1. gathered_nodes (List[NodeCoords]) The pre-gathered nodes to assign. + """ + pathsep: PathSeperators = kwargs.pop("pathsep", PathSeperators.AUTO) + anchor_name: str = kwargs.pop("anchor_name", "") + + if self.data is None: + self.logger.debug( + "Refusing to alias nodes in a null document!", + prefix="Processor::alias_gathered_nodes: ", data=self.data) + return + + anchor_node = self._get_anchor_node( + anchor_path, pathsep=pathsep, anchor_name=anchor_name) + + if gathered_nodes: + self._alias_nodes(gathered_nodes, anchor_node) + + def _alias_nodes( + self, gathered_nodes: List[NodeCoords], anchor_node: Any + ) -> None: + """ + Assign a YAML Anchor to its various YAML Alias nodes. + + Parameters: + 1. gathered_nodes (List[NodeCoords]) The pre-gathered nodes to assign. + 2. anchor_node (Any) The source YAML Anchor node. + + Returns: N/A + """ + anchor_name = anchor_node.anchor.value + for node_coord in gathered_nodes: + self.logger.debug( + "Attempting to set the anchor name for node to {}:" + .format(anchor_name), + data=node_coord, + prefix="yaml_set::_alias_nodes: ") + node_coord.parent[node_coord.parentref] = anchor_node + + def tag_nodes( + self, yaml_path: Union[YAMLPath, str], tag: str, **kwargs: Any + ) -> None: + """ + Gather and assign a data-type tag to nodes at YAML Path in data. + + Parameters: + 1. yaml_path (Union[YAMLPath, str]) The YAML Path to evaluate + 2. tag (str) The tag to assign + + Keyword Parameters: + * pathsep (PathSeperators) Forced YAML Path segment seperator; set + only when automatic inference fails; + default = PathSeperators.AUTO + + Returns: N/A + + Raises: + - `YAMLPathException` when YAML Path is invalid + """ + pathsep: PathSeperators = kwargs.pop("pathsep", PathSeperators.AUTO) + + if self.data is None: + self.logger.debug( + "Refusing to tag nodes from a null document!", + prefix="Processor::tag_nodes: ", data=self.data) + return + + if isinstance(yaml_path, str): + yaml_path = YAMLPath(yaml_path, pathsep) + elif pathsep is not PathSeperators.AUTO: + yaml_path.seperator = pathsep + + gathered_nodes: List[NodeCoords] = [] + for node_coords in self._get_required_nodes(self.data, yaml_path): + self.logger.debug( + "Gathered node for tagging:", + prefix="Processor::tag_nodes: ", data=node_coords) + gathered_nodes.append(node_coords) + + if len(gathered_nodes) > 0: + self.tag_gathered_nodes(gathered_nodes, tag) + + def tag_gathered_nodes( + self, gathered_nodes: List[NodeCoords], tag: str + ) -> None: + """Assign a data-type tag to a set of nodes.""" + # A YAML tag must be prefixed via at least one bang (!) + if tag and not tag[0] == "!": + tag = "!{}".format(tag) + + for node_coord in gathered_nodes: + old_node = node_coord.node + if node_coord.parent is None: + node_coord.node.yaml_set_tag(tag) + else: + node_coord.parent[node_coord.parentref] = Nodes.apply_yaml_tag( + node_coord.node, tag) + if Anchors.get_node_anchor(old_node) is not None: + Anchors.replace_anchor( + self.data, old_node, + node_coord.parent[node_coord.parentref]) + def delete_nodes(self, yaml_path: Union[YAMLPath, str], **kwargs: Any) -> Generator[NodeCoords, None, None]: """ - Delete nodes at YAML Path in data. + Gather and delete nodes at YAML Path in data. Parameters: 1. yaml_path (Union[YAMLPath, str]) The YAML Path to evaluate @@ -238,7 +474,7 @@ def delete_nodes(self, yaml_path: Union[YAMLPath, str], def delete_gathered_nodes(self, gathered_nodes: List[NodeCoords]) -> None: """ - Recursively delete pre-gathered nodes. + Delete pre-gathered nodes. Parameters: 1. gathered_nodes (List[NodeCoords]) The pre-gathered nodes to delete. diff --git a/yamlpath/wrappers/consoleprinter.py b/yamlpath/wrappers/consoleprinter.py index 7a4e9a37..55e9151b 100644 --- a/yamlpath/wrappers/consoleprinter.py +++ b/yamlpath/wrappers/consoleprinter.py @@ -251,7 +251,7 @@ def _debug_scalar(data: Any, **kwargs) -> str: print_anchor = kwargs.pop("print_anchor", True) print_tag = kwargs.pop("print_tag", True) print_type = kwargs.pop("print_type", False) - dtype = type(data) if print_type else "" + dtype: str = str(type(data)) if print_type else "" anchor_prefix = "" print_prefix = prefix @@ -272,6 +272,10 @@ def _debug_scalar(data: Any, **kwargs) -> str: if isinstance(data, TaggedScalar): dtype = "{}({})".format(dtype, type(data.value)) + # Report fold points, if present + if hasattr(data, "fold_pos"): + dtype += ",folded@{}".format(data.fold_pos) + print_prefix += anchor_prefix return ConsolePrinter._debug_prefix_lines( "{}{}{}".format(print_prefix, data, dtype))