From 59ec857cea908c4fc0c196f43af367b371350470 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Fri, 30 Jan 2015 13:33:37 +0000 Subject: [PATCH 01/32] Initial commit --- .gitignore | 0 LICENSE.md | 20 +++++++ README.md | 68 ++++++++++++++++++++++ macros/erlang_encode_macro.j2 | 63 ++++++++++++++++++++ macros/ini_encode_macro.j2 | 32 +++++++++++ macros/json_encode_macro.j2 | 77 +++++++++++++++++++++++++ macros/toml_encode_macro.j2 | 104 ++++++++++++++++++++++++++++++++++ macros/xml_encode_macro.j2 | 65 +++++++++++++++++++++ macros/yaml_encode_macro.j2 | 76 +++++++++++++++++++++++++ 9 files changed, 505 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE.md create mode 100644 README.md create mode 100644 macros/erlang_encode_macro.j2 create mode 100644 macros/ini_encode_macro.j2 create mode 100644 macros/json_encode_macro.j2 create mode 100644 macros/toml_encode_macro.j2 create mode 100644 macros/xml_encode_macro.j2 create mode 100644 macros/yaml_encode_macro.j2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..0d6d219 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) Jiri Tyr 2014 + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3270b85 --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +Encoder macros +============== + +Set of Jinja2 macros which help to encode Python data structure into a different +format. + + +Supported formats +----------------- + +- Erlang config +- INI +- JSON +- TOML +- XML +- YAML + + +Examples +-------- + +Erlang config format: + +``` +TODO +``` + +INI format: + +``` +TODO +``` + +JSON format: + +``` +TODO +``` + +TOML format: + +``` +TODO +``` + +XML format: + +``` +TODO +``` + +YAML format: + +``` +TODO +``` + + +License +------- + +MIT + + +Author +------ + +Jiri Tyr diff --git a/macros/erlang_encode_macro.j2 b/macros/erlang_encode_macro.j2 new file mode 100644 index 0000000..7a6d8c2 --- /dev/null +++ b/macros/erlang_encode_macro.j2 @@ -0,0 +1,63 @@ +{# + # Jinja2 macro which converts Python data structure to Erlang config format + #} + +{%- macro erlang_encode(item, level=0, indent=" ", detect_nums=false) %} + {%- if item is mapping %} + {#- The item is a dict #} + + {%- if item | length > 0 -%} + {{ "\n" }} + {%- endif %} + + {%- for key, val in item.iteritems() | sort -%} + {{ indent * (level+1) ~ "{" ~ key ~ ", "}}{{ erlang_encode( + val, level=level+1, indent=indent, detect_nums=detect_nums) }}{{ "}" }} + + {%- if loop.last -%} + {{ "\n" }} + {%- else -%} + {{ ",\n" }} + {%- endif %} + {%- endfor %} + + {%- elif item == "null" or item is number or (detect_nums and + ( + item | int | string == item or + item | float | string == item)) or + item == 'true' or + item == 'True' or + item == 'false' or + item == 'False'%} + + {#- Item is a value of a number, null or boolean -#} + + {{ item }} + + {%- elif item is string %} + {#- The item is a string -#} + + {{ '"' ~ item ~ '"' }} + + {%- else %} + {#- The item is a list -#} + + {{ "[" }} + + {%- for val in item -%} + {{ erlang_encode( + val, level=level+1, indent=indent, detect_nums=detect_nums) }} + + {%- if not loop.last -%} + {{ "," }} + {%- endif %} + {%- endfor -%} + + {%- if item | length > 0 -%} + {{ indent * level ~ "]" }} + {%- else -%} + {{ "]" }} + {%- endif %} + + {%- endif %} +{% endmacro %} diff --git a/macros/ini_encode_macro.j2 b/macros/ini_encode_macro.j2 new file mode 100644 index 0000000..aaa6824 --- /dev/null +++ b/macros/ini_encode_macro.j2 @@ -0,0 +1,32 @@ +{# + # Jinja2 macro which converts Python data structure to INI format + #} + +{%- macro ini_encode( + data, + uppercase_keys=false, + quote="", + delimiter="=", + first=[]) %} + + {#- First process vars without a section #} + {%- for key, value in data.iteritems() | sort %} + {%- if value is not mapping %} + {%- if first | length == 0 -%} + {{ "\n" }} + {%- if first.append(1) %}{% endif %} + {%- endif -%} + {{ (key | upper if uppercase_keys else key) ~ delimiter ~ quote ~ value ~ quote ~ "\n" }} + {%- endif %} + {%- endfor %} + + {#- Then process vars with a section #} + {%- for section, options in data.iteritems() | sort %} + {%- if options is mapping -%} + {{ "\n[" ~ section ~ "]\n" }} + {%- for key, value in options.iteritems() | sort -%} + {{ (key | upper if uppercase_keys else key) ~ delimiter ~ quote ~ value ~ quote ~ "\n" }} + {%- endfor %} + {%- endif %} + {%- endfor %} +{% endmacro %} diff --git a/macros/json_encode_macro.j2 b/macros/json_encode_macro.j2 new file mode 100644 index 0000000..3575898 --- /dev/null +++ b/macros/json_encode_macro.j2 @@ -0,0 +1,77 @@ +{# + # Jinja2 macro which converts Python data structure to JSON format + #} + +{%- macro json_encode(item, level=0, indent=" ", detect_nums=false) %} + {%- if item is mapping %} + {#- The item is a dict -#} + + {{ "{" }} + + {%- if item | length > 0 -%} + {{ "\n" }} + {%- endif %} + + {%- for key, val in item.iteritems() | sort -%} + {{ indent * (level+1) ~ '"' ~ key ~ '": '}}{{ json_encode( + val, level=level+1, indent=indent, detect_nums=detect_nums) }} + + {%- if loop.last -%} + {{ "\n" }} + {%- else -%} + {{ ",\n" }} + {%- endif %} + {%- endfor -%} + + {%- if item | length > 0 -%} + {{ indent * level ~ "}" }} + {%- else -%} + {{ "}" }} + {%- endif %} + + {%- elif item == "null" or item is number or (detect_nums and + ( + item | int | string == item or + item | float | string == item)) or + item == 'true' or + item == 'True' or + item == 'false' or + item == 'False'%} + + {#- Item is a value of a number, null or boolean -#} + + {{ item }} + + {%- elif item is string %} + {#- The item is a string -#} + + {{ '"' ~ item ~ '"' }} + + {%- else %} + {#- The item is a list -#} + + {{ "[" }} + + {%- if item | length > 0 -%} + {{ "\n" }} + {%- endif %} + + {%- for val in item -%} + {{ indent * (level+1) }}{{ json_encode( + val, level=level+1, indent=indent, detect_nums=detect_nums) }} + + {%- if loop.last -%} + {{ "\n" }} + {%- else -%} + {{ ",\n" }} + {%- endif %} + {%- endfor -%} + + {%- if item | length > 0 -%} + {{ indent * level ~ "]" }} + {%- else -%} + {{ "]" }} + {%- endif %} + + {%- endif %} +{% endmacro %} diff --git a/macros/toml_encode_macro.j2 b/macros/toml_encode_macro.j2 new file mode 100644 index 0000000..ab788c4 --- /dev/null +++ b/macros/toml_encode_macro.j2 @@ -0,0 +1,104 @@ +{# + # Jinja2 macro which converts Python data structure to TOML format + # (https://github.com/jtyr/yaml2toml-converter) + #} + +{%- macro yaml2toml(item, prevkey='', level=0, first=[], detect_nums=false) %} + {%- if item is mapping %} + {#- The item is a dict #} + + {#- First process all strings, numbers and lists #} + {%- for key, value in item.iteritems() | sort -%} + {%- if value is string or + value is number or + ( + value is not mapping and + value | length > 0 and + value[0] is not mapping) %} + + {#- The value is a string, a number, or a list -#} + {{ ' ' * level }}{{ key }}{{ " = " }}{{ yaml2toml( + value, + prevkey=prevkey, + level=level, + first=first, + detect_nums=detect_nums) }} + {%- if first.append(1) %}{% endif %} + {%- endif %} + {%- endfor %} + + {#- Then process all data structures #} + {%- for key, value in item.iteritems() | sort %} + {%- if value is not string and + value is not number and + ( + value is mapping or + value[0] is mapping) %} + + {%- set old_prevkey = prevkey %} + {%- set old_level = level %} + + {%- if value is mapping %} + {#- The value is a dict #} + {%- if prevkey != '' and prevkey != key %} + {%- set level = level + 1 %} + {%- endif %} + + {%- set prevkey = (key if prevkey == '' else prevkey + '.' + key) %} + + {%- if first | length > 0 -%} + {{ '\n' }} + {%- endif -%} + + {{ ' ' * level }}[{{ prevkey }}]{{ "\n" }} + {%- elif value[0] is mapping %} + {#- The value is a table #} + {%- set prevkey = (key if prevkey == '' else prevkey + '.' + key) %} + {%- set level = level +1 %} + {%- endif -%} + + {{ yaml2toml( + value, + prevkey=prevkey, + level=level, + first=first, + detect_nums=detect_nums) }} + + {%- set prevkey = old_prevkey %} + {%- set level = old_level %} + {%- if first.append(1) %}{% endif %} + {%- endif %} + {%- endfor %} + + {%- elif item is number or + (detect_nums and + ( + item | int | string == item or + item | float | string == item)) or + item == 'true' or + item == 'True' or + item == 'false' or + item == 'False' %} + + {#- The item is a number or boolean -#} + {{ item }}{{ "\n" }} + + {%- elif item is string %} + {#- The item is a string -#} + "{{ item | replace("\n", "\\n") | replace("\t", "\\t") }}"{{ "\n" }} + + {%- else %} + {#- The item is a list #} + {%- if item[0] is mapping %} + {%- for d in item -%} + {{ "\n" }}{{ ' ' * level }}[[{{ prevkey }}]]{{ "\n" }}{{ yaml2toml( + d, + level=level, + first=first, + detect_nums=detect_nums) }} + {%- endfor %} + {%- else -%} + {{ item }}{{ "\n" }} + {%- endif %} + {%- endif %} +{%- endmacro -%} diff --git a/macros/xml_encode_macro.j2 b/macros/xml_encode_macro.j2 new file mode 100644 index 0000000..001f436 --- /dev/null +++ b/macros/xml_encode_macro.j2 @@ -0,0 +1,65 @@ +{# + # Jinja2 macro which converts Python data structure to XML format + # + # Limitation: + # - can not define attributes in a list + # - can not use variables in attributes + #} + +{%- macro xml_encode(item, level=0, indent=" ", first=[], prev_name=none) %} + {%- if item is mapping %} + {#- The item is a dict #} + + {%- set prev_list = [0] %} + + {%- for key, val in item.iteritems() | sort -%} + {%- if first | length > 0 and prev_list[0] == 0 -%} + {{ "\n" }} + {%- endif -%} + + {%- if first.append(0) %}{% endif %} + {%- if prev_list.insert(0, 0) and prev_list.remove(1) %}{% endif %} + + {%- if val is none -%} + {{ indent * level }}<{{ key }} /> + {%- elif val is sequence and val is not string and val is not mapping -%} + {#- Only if val is a list -#} + {{ xml_encode(val, level=level, indent=indent, prev_name=key) }} + {%- if prev_list.insert(0, 1) and prev_list.remove(1) %}{% endif %} + {%- else -%} + {{ indent * level }}<{{ key }}>{{ xml_encode(val, level=level+1, indent=indent) }} + + {%- if val is not string and val is not number and val is not none -%} + {{ indent * level }} + {%- endif -%} + + + + {%- if loop.last-%} + {{ "\n" }} + {%- endif %} + {%- endif %} + {%- endfor %} + + {%- elif item is number or item is string %} + {#- The item is a number, string or boolean -#} + + {{ item }} + + {%- elif item is none %} + {#- Item is a value of an empty element #} + + {%- else %} + {#- The item is a list #} + + {%- for e in item -%} + {{ indent * level }}<{{ prev_name }}>{{ xml_encode(e, level=level+1, indent=indent, prev_name=prev_name) }} + + {%- if e is not string and e is not number and e is not none -%} + {{ indent * level }} + {%- endif -%} + + {{ "\n" }} + {%- endfor %} + {%- endif %} +{%- endmacro -%} diff --git a/macros/yaml_encode_macro.j2 b/macros/yaml_encode_macro.j2 new file mode 100644 index 0000000..1d32834 --- /dev/null +++ b/macros/yaml_encode_macro.j2 @@ -0,0 +1,76 @@ +{# + # Jinja2 macro which converts Python data structure to YAML format + #} + +{%- macro yaml_encode( + item, + level=0, + skip_indent=false, + quotation_mark='"', + indention_unit=' ', + detect_nums=false) %} + + {%- if item is mapping %} + {#- The item is a dict #} + {%- for key, value in item.iteritems() | sort -%} + {%- if skip_indent and loop.first -%} + {{ ' ' }} + {%- else -%} + {{ indention_unit * level }} + {%- endif -%} + + {{ key }}: + + {%- if value is not string and value is not number -%} + {{ "\n" }} + {%- endif -%} + + {{ yaml_encode( + value, + level=level+1, + quotation_mark=quotation_mark, + indention_unit=indention_unit, + detect_nums=detect_nums) }} + {%- else -%} + {{ indention_unit * level + '{ }\n' }} + {%- endfor %} + + {%- elif item is number or + (detect_nums and + ( + item | int | string == item or + item | float | string == item)) or + item == 'true' or + item == 'True' or + item == 'false' or + item == 'False' %} + + {#- The item is a number or boolean -#} + {{ ' ' ~ item | lower ~ "\n" }} + + {%- elif item is string %} + {#- The item is a string -#} + {{ ' ' + + quotation_mark + + item | replace(quotation_mark, '\\' + quotation_mark) + + quotation_mark + "\n" }} + + {%- else %} + {#- The item is a list #} + {%- for n in item -%} + {{ indention_unit * level }}- + {%- if n is not string and + n is not number and + n is not mapping -%} + {{ "\n" }} + {%- endif -%} + {{ yaml_encode( + n, + level=level+1, + skip_indent=true, + quotation_mark=quotation_mark, + indention_unit=indention_unit, + detect_nums=detect_nums) }} + {%- endfor %} + {%- endif %} +{% endmacro %} From 94f11572679fb13828d19e531a9bbefb6a3d895c Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Mon, 16 Feb 2015 19:00:20 +0000 Subject: [PATCH 02/32] Documentation and minor macros improvements --- LICENSE.md | 2 +- README.md | 305 ++++++++++++++++++++++++++++++++-- TODO.md | 7 + hosts | 1 + macros/erlang_encode_macro.j2 | 63 +++++-- macros/ini_encode_macro.j2 | 28 ++-- macros/json_encode_macro.j2 | 34 ++-- macros/toml_encode_macro.j2 | 72 ++++---- macros/xml_encode_macro.j2 | 33 ++-- macros/yaml_encode_macro.j2 | 61 +++---- site.yaml | 73 ++++++++ templates/test.erlang.j2 | 9 + templates/test.ini.j2 | 9 + templates/test.json.j2 | 3 + templates/test.toml.j2 | 9 + templates/test.xml.j2 | 13 ++ templates/test.yaml.j2 | 9 + vars/erlang_test.yaml | 14 ++ vars/ini_test.yaml | 13 ++ vars/json_test.yaml | 16 ++ vars/toml_test.yaml | 33 ++++ vars/xml_test.yaml | 15 ++ vars/yaml_test.yaml | 16 ++ yaml_converter.py | 67 ++++++++ 24 files changed, 779 insertions(+), 126 deletions(-) create mode 100644 TODO.md create mode 100644 hosts create mode 100644 site.yaml create mode 100644 templates/test.erlang.j2 create mode 100644 templates/test.ini.j2 create mode 100644 templates/test.json.j2 create mode 100644 templates/test.toml.j2 create mode 100644 templates/test.xml.j2 create mode 100644 templates/test.yaml.j2 create mode 100644 vars/erlang_test.yaml create mode 100644 vars/ini_test.yaml create mode 100644 vars/json_test.yaml create mode 100644 vars/toml_test.yaml create mode 100644 vars/xml_test.yaml create mode 100644 vars/yaml_test.yaml create mode 100755 yaml_converter.py diff --git a/LICENSE.md b/LICENSE.md index 0d6d219..ce081ff 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) Jiri Tyr 2014 +Copyright (c) Jiri Tyr 2015 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/README.md b/README.md index 3270b85..2dac453 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,21 @@ -Encoder macros -============== +Jinja2 Encoder Macros +===================== Set of Jinja2 macros which help to encode Python data structure into a different -format. +file format. + + +Motivation +---------- + +The motivation to create these macros was to be able to create an universal +description of configuration files in [Ansible](http://ansible.com). Ansible is +using YAML format as the main data input which allows to describe almost any +structure of a configuration file. Ansible converts the YAML data into a Python +data structure using primitive data types (e.g. `dict`, `list`, `string`, +`number`, `boolean`, ...). That data structure is then used in Jinja2 template +files to produce the final configuration file. The encoder macros are used to +convert the Python data structure to another format. Supported formats @@ -16,43 +29,301 @@ Supported formats - YAML -Examples --------- +Ansible example +--------------- -Erlang config format: +Clone this repo into the directory where your playbook is. Then use the macros +from your playbook (e.g. `site.yaml`) as follows: ``` -TODO +# Playbook example +- name: Play to test the encoder macros + hosts: all + vars: + data: + var1: val1 + var2: val2 + section1: + aaa: asdf + bbb: 123 + ccc: 'true' + section2: + ddd: asdfasd + eee: 1234 + fff: 'false' + tasks: + - name: Create test.ini file + template: + src: templates/test.ini.j2 + dest: /tmp/test.ini ``` -INI format: +Where the `templates/test.ini.j2` contains: ``` -TODO +# +# This file is managed by Ansible. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +{% from 'macros/ini_encode_macro.j2' import ini_encode with context -%} + +{{ ini_encode(data) }} ``` -JSON format: +And if you call the playbook: ``` -TODO +$ ansible-playbook -i hosts ./site.yaml ``` -TOML format: +You get the following in the `/tmp/test.ini`: ``` -TODO +# +# This file is managed by Ansible. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +var1=val1 +var2=val2 + +[section1] +aaa=asdf +bbb=123 +ccc=true + +[section2] +ddd=asdfasd +eee=1234 +fff=false ``` -XML format: +Look into the `site.yaml` and `templates\*.j2` for more examples. + + +Macros parameters +----------------- + +The first parameter passed to the macro is always the data structure. Any +following parameter should be addressed by a keyword (e.g. `indent=" "` where +`indent` is the keyword). + + +### `erlang_encode()` + +- `item` + > Variable holding the input data for the macro. +- `convert_bools=true` + > Indicates whether Boolean values presented as a string should be converted + > to a real Booblean value. For example `var1: 'True'` would be represented + > as a string but using the `convert_bools=true` will convert it to a Boolean + > like it would be defined like this: `var1: true`. +- `convert_nums=false` + > Indicates whether number presented as a string should be converted to + > number. For example `var1: '123'` would be represented as a string but + > using the `convert_nums=true` will convert it to a number like it would + > be defined like this: `var1: 123`. +- `first=[]` + > Indicates whether the item being processed is actually a first item. It's + > used only internally in the macro. +- `indent=" "` + > Defines the indentation unit. +- `level=0` + > Indicates the initial level of the indentation. Value `0` starts indenting + > from the beginning of the line. Setting the value to higher than `0` + > indents the content by `intent * level`. + + +### `ini_encode()` +- `item` + > Variable holding the input data for the macro. +- `delimiter="="` + > Sign separating the *property* and the *value*. By default it's set to `=` + > but it can also be set to ` = `. +- `first=[]` + > Indicates whether the item being processed is actually a first item. It's + > used only internally in the macro. +- `quote=""` + > Allows to set the *value* quoting. Use `quote="'"` or `quote='"'`. +- `ucase_prop=false` + > Indicates whether the *property* should be made uppercase. + + +### `json_encode()` +- `item` + > Variable holding the input data for the macro. +- `convert_bools=true` + > Indicates whether Boolean values presented as a string should be converted + > to a real Booblean value. For example `var1: 'True'` would be represented + > as a string but using the `convert_bools=true` will convert it to a Boolean + > like it would be defined like this: `var1: true`. +- `convert_nums=false` + > Indicates whether number presented as a string should be converted to + > number. For example `var1: '123'` would be represented as a string but + > using the `convert_nums=true` will convert it to a number like it would + > be defined like this: `var1: 123`. +- `indent=" "` + > Defines the indentation unit. +- `level=0` + > Indicates the initial level of the indentation. Value `0` starts indenting + > from the beginning of the line. Setting the value to higher than `0` + > indents the content by `intent * level`. + + +### `toml_encode()` + +- `item` + > Variable holding the input data for the macro. +- `convert_bools=true` + > Indicates whether Boolean values presented as a string should be converted + > to a real Booblean value. For example `var1: 'True'` would be represented + > as a string but using the `convert_bools=true` will convert it to a Boolean + > like it would be defined like this: `var1: true`. +- `convert_nums=false` + > Indicates whether number presented as a string should be converted to + > number. For example `var1: '123'` would be represented as a string but + > using the `convert_nums=true` will convert it to a number like it would + > be defined like this: `var1: 123`. +- `first=[]` + > Indicates whether the item being processed is actually a first item. It's + > used only internally in the macro. +- `indent=" "` + > Defines the indentation unit. +- `level=0` + > Indicates the initial level of the indentation. Value `0` starts indenting + > from the beginning of the line. Setting the value to higher than `0` + > indents the content by `intent * level`. +- `prevkey=""` + > Defines the name of the previous key in the recursive macro calls. It's used + > only internally in the macro. + + +### `xml_encode()` +- `item` + > Variable holding the input data for the macro. +- `first=[]` + > Indicates whether the item being processed is actually a first item. It's + > used only internally in the macro. +- `indent=" "` + > Defines the indentation unit. +- `level=0` + > Indicates the initial level of the indentation. Value `0` starts indenting + > from the beginning of the line. Setting the value to higher than `0` + > indents the content by `intent * level`. +- `prevkey=none` + > Defines the name of the previous key in the recursive macro calls. It's used + > only internally in the macro. + + +### `yaml_encode()` +- `item` + > Variable holding the input data for the macro. +- `convert_bools=true` + > Indicates whether Boolean values presented as a string should be converted + > to a real Booblean value. For example `var1: 'True'` would be represented + > as a string but using the `convert_bools=true` will convert it to a Boolean + > like it would be defined like this: `var1: true`. +- `convert_nums=false` + > Indicates whether number presented as a string should be converted to + > number. For example `var1: '123'` would be represented as a string but + > using the `convert_nums=true` will convert it to a number like it would + > be defined like this: `var1: 123`. +- `indent=" "` + > Defines the indentation unit. +- `level=0` + > Indicates the initial level of the indentation. Value `0` starts indenting + > from the beginning of the line. Setting the value to higher than `0` + > indents the content by `intent * level`. +- `quote='"'` + > Allows to set the *value* quoting. Use `quote="'"` or `quote='"'`. +- `skip_indent=false` + > Indicates whether the indentation should be skipped for the specific item. + > It's used only internally in the macro. + + +Limitations +----------- + +### Erlang, JSON, TOML and YAML macros + +In these sensitive file formats, the Boolean values represented as string +(`"true"`, `"True"`, `"false"` and `"False"`) are automatically converted to +Boolean value (`true` and `false`). The reason for this behavior is that if you +want to pass a Boolean variable in Ansible, you have to actually quote it (e.g. +`var1: "{{ my_boolean_var }}"`). Once the variable is quoted, it's not a Boolean +variable any more. If you want to use Boolean as a string in your final +configuration file, try to use value with a space (e.g. `' true'`). + + +### XML macro + +YAML doesn't allow to fully map XML data with attributes. This is why XML +attributes must be defined as part of the element name (e.g. `'elem attr1="val1" +attr2="val2"': Value of the element`). This is also the reason why elements of +the same name can only have the same set of attributes. For example the following YAML +input data: + +``` +--- +root: + 'elem attr1="val1" attr2="val2"': + - element value1 + - element value2 +``` + +Will produce the following XML file (`elem` has the same set of attributes): ``` -TODO + + element value1 + element value2 + +``` + +Another limitation of the XML macro is that attributes can not have value +derivated from a Jinja2 variable in the Ansible context. + + +Python example +-------------- + +Although the macros are intended to be used primarily in Ansible, they can also +be used directly from Python: + +``` +from jinja2 import Template +from yaml import load +import sys + +# Load the YAML data to Python data structure +yaml_fh = open('vars/ini_test.yaml') +yaml_data = load(yaml_fh) +yaml_fh.close() + +# Take only the content of the data variable +yaml_data = yaml_data['data'] + +# Read Jinja2 template as text +template_fh = open('macros/ini_encode_macro.j2') +template_text = template_fh.read() +template_fh.close() + +# Create Jinja2 template object +t = Template(template_text) +# Convert the YAML data to INI format +output = t.module.ini_encode(yaml_data).encode('utf8') + +# Print the result +sys.stdout.write(output) ``` -YAML format: +Or you can use the `yaml_converter.py`: ``` -TODO +$ ./yaml_converter.py -f json -v data -y ./vars/json_test.yaml ``` diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..627f68b --- /dev/null +++ b/TODO.md @@ -0,0 +1,7 @@ +TODO +==== + +- Fix formating when passing real Boolean value (True -> true) in all + Boolean-sensitive formats. Probably requires a new section. +- Missing new line at the end of Erlang and JSON file generated by the Python + script diff --git a/hosts b/hosts new file mode 100644 index 0000000..8635629 --- /dev/null +++ b/hosts @@ -0,0 +1 @@ +localhost ansible_python_interpreter=/usr/bin/python2 ansible_connection=local diff --git a/macros/erlang_encode_macro.j2 b/macros/erlang_encode_macro.j2 index 7a6d8c2..022050f 100644 --- a/macros/erlang_encode_macro.j2 +++ b/macros/erlang_encode_macro.j2 @@ -2,33 +2,51 @@ # Jinja2 macro which converts Python data structure to Erlang config format #} -{%- macro erlang_encode(item, level=0, indent=" ", detect_nums=false) %} +{%- macro erlang_encode( + item, + convert_bools=true, + convert_nums=false, + first=[], + indent=" ", + level=0) %} + {%- if item is mapping %} {#- The item is a dict #} - {%- if item | length > 0 -%} + {%- if first | length > 0 and item | length > 0 -%} {{ "\n" }} {%- endif %} + {%- if first.append(1) %}{% endif %} + {%- for key, val in item.iteritems() | sort -%} - {{ indent * (level+1) ~ "{" ~ key ~ ", "}}{{ erlang_encode( - val, level=level+1, indent=indent, detect_nums=detect_nums) }}{{ "}" }} + {{ indent * level ~ "{" ~ key ~ ","}} - {%- if loop.last -%} - {{ "\n" }} - {%- else -%} + {%- if val is not mapping -%} + {{ " " }} + {%- endif -%} + + {{ erlang_encode( + val, + convert_bools=convert_bools, + convert_nums=convert_nums, + first=first, + indent=indent, + level=level+1) }}{{ "}" }} + + {%- if not loop.last -%} {{ ",\n" }} {%- endif %} {%- endfor %} - {%- elif item == "null" or item is number or (detect_nums and - ( + {%- elif item == "null" or item is number or + (convert_nums and ( item | int | string == item or item | float | string == item)) or - item == 'true' or - item == 'True' or - item == 'false' or - item == 'False'%} + item == "true" or + item == "True" or + item == "false" or + item == "False" %} {#- Item is a value of a number, null or boolean -#} @@ -45,19 +63,30 @@ {{ "[" }} {%- for val in item -%} + {%- if val is string or val is number -%} + {{ "\n" ~ indent * (level) }} + {%- endif -%} + {{ erlang_encode( - val, level=level+1, indent=indent, detect_nums=detect_nums) }} + val, + convert_bools=convert_bools, + convert_nums=convert_nums, + first=first, + indent=indent, + level=level+1) }} - {%- if not loop.last -%} + {%- if loop.last -%} + {{ "\n" }} + {%- else -%} {{ "," }} {%- endif %} {%- endfor -%} {%- if item | length > 0 -%} - {{ indent * level ~ "]" }} + {{ indent * (level-1) ~ "]" }} {%- else -%} {{ "]" }} {%- endif %} {%- endif %} -{% endmacro %} +{%- endmacro -%} diff --git a/macros/ini_encode_macro.j2 b/macros/ini_encode_macro.j2 index aaa6824..dcc5a39 100644 --- a/macros/ini_encode_macro.j2 +++ b/macros/ini_encode_macro.j2 @@ -3,30 +3,34 @@ #} {%- macro ini_encode( - data, - uppercase_keys=false, - quote="", + item, delimiter="=", - first=[]) %} + first=[], + quote="", + ucase_prop=false) %} {#- First process vars without a section #} - {%- for key, value in data.iteritems() | sort %} + {%- for property, value in item.iteritems() | sort %} {%- if value is not mapping %} {%- if first | length == 0 -%} - {{ "\n" }} {%- if first.append(1) %}{% endif %} {%- endif -%} - {{ (key | upper if uppercase_keys else key) ~ delimiter ~ quote ~ value ~ quote ~ "\n" }} + {{ (property | upper if ucase_prop else property) ~ delimiter ~ quote ~ value ~ quote ~ "\n" }} {%- endif %} {%- endfor %} {#- Then process vars with a section #} - {%- for section, options in data.iteritems() | sort %} + {%- for section, options in item.iteritems() | sort %} {%- if options is mapping -%} - {{ "\n[" ~ section ~ "]\n" }} - {%- for key, value in options.iteritems() | sort -%} - {{ (key | upper if uppercase_keys else key) ~ delimiter ~ quote ~ value ~ quote ~ "\n" }} + {%- if first | length == 0 -%} + {%- if first.append(1) %}{% endif %} + {%- else -%} + {{ "\n" }} + {%- endif -%} + {{ "[" ~ section ~ "]\n" }} + {%- for property, value in options.iteritems() | sort -%} + {{ (property | upper if ucase_prop else property) ~ delimiter ~ quote ~ value ~ quote ~ "\n" }} {%- endfor %} {%- endif %} {%- endfor %} -{% endmacro %} +{%- endmacro -%} diff --git a/macros/json_encode_macro.j2 b/macros/json_encode_macro.j2 index 3575898..f44d577 100644 --- a/macros/json_encode_macro.j2 +++ b/macros/json_encode_macro.j2 @@ -2,7 +2,13 @@ # Jinja2 macro which converts Python data structure to JSON format #} -{%- macro json_encode(item, level=0, indent=" ", detect_nums=false) %} +{%- macro json_encode( + item, + convert_bools=true, + convert_nums=false, + indent=" ", + level=0) %} + {%- if item is mapping %} {#- The item is a dict -#} @@ -14,7 +20,11 @@ {%- for key, val in item.iteritems() | sort -%} {{ indent * (level+1) ~ '"' ~ key ~ '": '}}{{ json_encode( - val, level=level+1, indent=indent, detect_nums=detect_nums) }} + val, + convert_bools=convert_bools, + convert_nums=convert_nums, + indent=indent, + level=level+1) }} {%- if loop.last -%} {{ "\n" }} @@ -29,14 +39,14 @@ {{ "}" }} {%- endif %} - {%- elif item == "null" or item is number or (detect_nums and - ( + {%- elif item == "null" or item is number or + (convert_nums and ( item | int | string == item or item | float | string == item)) or - item == 'true' or - item == 'True' or - item == 'false' or - item == 'False'%} + item == "true" or + item == "True" or + item == "false" or + item == "False" %} {#- Item is a value of a number, null or boolean -#} @@ -58,7 +68,11 @@ {%- for val in item -%} {{ indent * (level+1) }}{{ json_encode( - val, level=level+1, indent=indent, detect_nums=detect_nums) }} + val, + convert_bools=convert_bools, + convert_nums=convert_nums, + indent=indent, + level=level+1) }} {%- if loop.last -%} {{ "\n" }} @@ -74,4 +88,4 @@ {%- endif %} {%- endif %} -{% endmacro %} +{%- endmacro -%} diff --git a/macros/toml_encode_macro.j2 b/macros/toml_encode_macro.j2 index ab788c4..61e6a31 100644 --- a/macros/toml_encode_macro.j2 +++ b/macros/toml_encode_macro.j2 @@ -1,9 +1,16 @@ {# # Jinja2 macro which converts Python data structure to TOML format - # (https://github.com/jtyr/yaml2toml-converter) #} -{%- macro yaml2toml(item, prevkey='', level=0, first=[], detect_nums=false) %} +{%- macro toml_encode( + item, + convert_bools=true, + convert_nums=false, + first=[], + indent=" ", + level=0, + prevkey="") %} + {%- if item is mapping %} {#- The item is a dict #} @@ -17,12 +24,14 @@ value[0] is not mapping) %} {#- The value is a string, a number, or a list -#} - {{ ' ' * level }}{{ key }}{{ " = " }}{{ yaml2toml( - value, - prevkey=prevkey, - level=level, - first=first, - detect_nums=detect_nums) }} + {{ indent * level }}{{ key }}{{ " = " }}{{ toml_encode( + value, + convert_bools=convert_bools, + convert_nums=convert_nums, + first=first, + indent=indent, + level=level, + prevkey=prevkey) }} {%- if first.append(1) %}{% endif %} {%- endif %} {%- endfor %} @@ -40,29 +49,31 @@ {%- if value is mapping %} {#- The value is a dict #} - {%- if prevkey != '' and prevkey != key %} + {%- if prevkey != "" and prevkey != key %} {%- set level = level + 1 %} {%- endif %} - {%- set prevkey = (key if prevkey == '' else prevkey + '.' + key) %} + {%- set prevkey = (key if prevkey == "" else prevkey + "." + key) %} {%- if first | length > 0 -%} - {{ '\n' }} + {{ "\n" }} {%- endif -%} - {{ ' ' * level }}[{{ prevkey }}]{{ "\n" }} + {{ indent * level }}[{{ prevkey }}]{{ "\n" }} {%- elif value[0] is mapping %} {#- The value is a table #} - {%- set prevkey = (key if prevkey == '' else prevkey + '.' + key) %} + {%- set prevkey = (key if prevkey == "" else prevkey + "." + key) %} {%- set level = level +1 %} {%- endif -%} - {{ yaml2toml( - value, - prevkey=prevkey, - level=level, - first=first, - detect_nums=detect_nums) }} + {{ toml_encode( + value, + convert_bools=convert_bools, + convert_nums=convert_nums, + first=first, + indent=indent, + level=level, + prevkey=prevkey) }} {%- set prevkey = old_prevkey %} {%- set level = old_level %} @@ -71,14 +82,13 @@ {%- endfor %} {%- elif item is number or - (detect_nums and - ( + (convert_nums and ( item | int | string == item or item | float | string == item)) or - item == 'true' or - item == 'True' or - item == 'false' or - item == 'False' %} + item == "true" or + item == "True" or + item == "false" or + item == "False" %} {#- The item is a number or boolean -#} {{ item }}{{ "\n" }} @@ -91,11 +101,13 @@ {#- The item is a list #} {%- if item[0] is mapping %} {%- for d in item -%} - {{ "\n" }}{{ ' ' * level }}[[{{ prevkey }}]]{{ "\n" }}{{ yaml2toml( - d, - level=level, - first=first, - detect_nums=detect_nums) }} + {{ "\n" }}{{ indent * level }}[[{{ prevkey }}]]{{ "\n" }}{{ toml_encode( + d, + convert_bools=convert_bools, + convert_nums=convert_nums, + first=first, + indent=indent, + level=level) }} {%- endfor %} {%- else -%} {{ item }}{{ "\n" }} diff --git a/macros/xml_encode_macro.j2 b/macros/xml_encode_macro.j2 index 001f436..15e7d9b 100644 --- a/macros/xml_encode_macro.j2 +++ b/macros/xml_encode_macro.j2 @@ -1,12 +1,14 @@ {# # Jinja2 macro which converts Python data structure to XML format - # - # Limitation: - # - can not define attributes in a list - # - can not use variables in attributes #} -{%- macro xml_encode(item, level=0, indent=" ", first=[], prev_name=none) %} +{%- macro xml_encode( + item, + first=[], + indent=" ", + level=0, + prevkey=none) %} + {%- if item is mapping %} {#- The item is a dict #} @@ -24,16 +26,23 @@ {{ indent * level }}<{{ key }} /> {%- elif val is sequence and val is not string and val is not mapping -%} {#- Only if val is a list -#} - {{ xml_encode(val, level=level, indent=indent, prev_name=key) }} + {{ xml_encode( + val, + indent=indent, + level=level, + prevkey=key) }} {%- if prev_list.insert(0, 1) and prev_list.remove(1) %}{% endif %} {%- else -%} - {{ indent * level }}<{{ key }}>{{ xml_encode(val, level=level+1, indent=indent) }} + {{ indent * level }}<{{ key }}>{{ xml_encode( + val, + indent=indent, + level=level+1) }} {%- if val is not string and val is not number and val is not none -%} {{ indent * level }} {%- endif -%} - + {%- if loop.last-%} {{ "\n" }} @@ -53,13 +62,17 @@ {#- The item is a list #} {%- for e in item -%} - {{ indent * level }}<{{ prev_name }}>{{ xml_encode(e, level=level+1, indent=indent, prev_name=prev_name) }} + {{ indent * level }}<{{ prevkey }}>{{ xml_encode( + e, + indent=indent, + level=level+1, + prevkey=prevkey) }} {%- if e is not string and e is not number and e is not none -%} {{ indent * level }} {%- endif -%} - {{ "\n" }} + {{ "\n" }} {%- endfor %} {%- endif %} {%- endmacro -%} diff --git a/macros/yaml_encode_macro.j2 b/macros/yaml_encode_macro.j2 index 1d32834..39c8561 100644 --- a/macros/yaml_encode_macro.j2 +++ b/macros/yaml_encode_macro.j2 @@ -3,20 +3,21 @@ #} {%- macro yaml_encode( - item, - level=0, - skip_indent=false, - quotation_mark='"', - indention_unit=' ', - detect_nums=false) %} + item, + convert_bools=true, + convert_nums=false, + indent=" ", + level=0, + quote='"', + skip_indent=false) %} {%- if item is mapping %} {#- The item is a dict #} {%- for key, value in item.iteritems() | sort -%} {%- if skip_indent and loop.first -%} - {{ ' ' }} + {{ " " }} {%- else -%} - {{ indention_unit * level }} + {{ indent * level }} {%- endif -%} {{ key }}: @@ -27,38 +28,39 @@ {{ yaml_encode( value, + convert_bools=convert_bools, + convert_nums=convert_nums, + indent=indent, level=level+1, - quotation_mark=quotation_mark, - indention_unit=indention_unit, - detect_nums=detect_nums) }} + quote=quote) }} {%- else -%} - {{ indention_unit * level + '{ }\n' }} + {{ indent * level + "{ }\n" }} {%- endfor %} {%- elif item is number or - (detect_nums and - ( + (convert_nums and ( item | int | string == item or item | float | string == item)) or - item == 'true' or - item == 'True' or - item == 'false' or - item == 'False' %} + (convert_bools and ( + item == "true" or + item == "True" or + item == "false" or + item == "False")) %} {#- The item is a number or boolean -#} - {{ ' ' ~ item | lower ~ "\n" }} + {{ " " ~ item | lower ~ "\n" }} {%- elif item is string %} {#- The item is a string -#} - {{ ' ' + - quotation_mark + - item | replace(quotation_mark, '\\' + quotation_mark) + - quotation_mark + "\n" }} + {{ " " + + quote + + item | replace(quote, "\\" + quote) + + quote + "\n" }} {%- else %} {#- The item is a list #} {%- for n in item -%} - {{ indention_unit * level }}- + {{ indent * level }}- {%- if n is not string and n is not number and n is not mapping -%} @@ -66,11 +68,12 @@ {%- endif -%} {{ yaml_encode( n, + convert_bools=convert_bools, + convert_nums=convert_nums, + indent=indent, level=level+1, - skip_indent=true, - quotation_mark=quotation_mark, - indention_unit=indention_unit, - detect_nums=detect_nums) }} + quote=quote, + skip_indent=true) }} {%- endfor %} {%- endif %} -{% endmacro %} +{%- endmacro -%} diff --git a/site.yaml b/site.yaml new file mode 100644 index 0000000..d85705f --- /dev/null +++ b/site.yaml @@ -0,0 +1,73 @@ +--- + +- name: Erlang encode example + hosts: all + vars_files: + - vars/erlang_test.yaml + tasks: + - name: Create test.erlang file + template: + src: templates/test.erlang.j2 + dest: /tmp/test.erlang + tags: + - erlang + +- name: INI encode example + hosts: all + vars_files: + - vars/ini_test.yaml + tasks: + - name: Create test.ini file + template: + src: templates/test.ini.j2 + dest: /tmp/test.ini + tags: + - ini + +- name: JSON encode example + hosts: all + vars_files: + - vars/json_test.yaml + tasks: + - name: Create test.json file + template: + src: templates/test.json.j2 + dest: /tmp/test.json + tags: + - json + +- name: TOML encode example + hosts: all + vars_files: + - vars/toml_test.yaml + tasks: + - name: Create test.toml file + template: + src: templates/test.toml.j2 + dest: /tmp/test.toml + tags: + - toml + +- name: XML encode example + hosts: all + vars_files: + - vars/xml_test.yaml + tasks: + - name: Create test.xml file + template: + src: templates/test.xml.j2 + dest: /tmp/test.xml + tags: + - xml + +- name: YAML encode example + hosts: all + vars_files: + - vars/yaml_test.yaml + tasks: + - name: Create test.yaml file + template: + src: templates/test.yaml.j2 + dest: /tmp/test.yaml + tags: + - yaml diff --git a/templates/test.erlang.j2 b/templates/test.erlang.j2 new file mode 100644 index 0000000..8f17d2a --- /dev/null +++ b/templates/test.erlang.j2 @@ -0,0 +1,9 @@ +%% +%% This file is managed by Ansible. +%% Do not edit this file manually. +%% Any changes will be automatically reverted. +%% + +{% from 'macros/erlang_encode_macro.j2' import erlang_encode with context -%} + +{{ erlang_encode(data, convert_nums=true) }} diff --git a/templates/test.ini.j2 b/templates/test.ini.j2 new file mode 100644 index 0000000..d1d0363 --- /dev/null +++ b/templates/test.ini.j2 @@ -0,0 +1,9 @@ +# +# This file is managed by Ansible. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +{% from 'macros/ini_encode_macro.j2' import ini_encode with context -%} + +{{ ini_encode(data) }} diff --git a/templates/test.json.j2 b/templates/test.json.j2 new file mode 100644 index 0000000..159b0b2 --- /dev/null +++ b/templates/test.json.j2 @@ -0,0 +1,3 @@ +{% from 'macros/json_encode_macro.j2' import json_encode with context -%} + +{{ json_encode(data, convert_nums=true) }} diff --git a/templates/test.toml.j2 b/templates/test.toml.j2 new file mode 100644 index 0000000..5aa3d29 --- /dev/null +++ b/templates/test.toml.j2 @@ -0,0 +1,9 @@ +# +# This file is managed by Ansible. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +{% from 'macros/toml_encode_macro.j2' import toml_encode with context -%} + +{{ toml_encode(data) }} diff --git a/templates/test.xml.j2 b/templates/test.xml.j2 new file mode 100644 index 0000000..d63814d --- /dev/null +++ b/templates/test.xml.j2 @@ -0,0 +1,13 @@ + + + + +{% from 'macros/xml_encode_macro.j2' import xml_encode with context -%} + +{{ xml_encode(data) }} diff --git a/templates/test.yaml.j2 b/templates/test.yaml.j2 new file mode 100644 index 0000000..cddd19c --- /dev/null +++ b/templates/test.yaml.j2 @@ -0,0 +1,9 @@ +# +# This file is managed by Ansible. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +{% from 'macros/yaml_encode_macro.j2' import yaml_encode with context -%} + +{{ yaml_encode(data, convert_nums=true) }} diff --git a/vars/erlang_test.yaml b/vars/erlang_test.yaml new file mode 100644 index 0000000..f0980dc --- /dev/null +++ b/vars/erlang_test.yaml @@ -0,0 +1,14 @@ +--- + +data: + rabbit: + tcp_listeners: + - 127.0.0.1: 5672 + ssl_listeners: + - 5671 + ssl_options: + - cacertfile: /path/to/testca/cacert.pem + - certfile: /path/to/server/cert.pem + - keyfile: /path/to/server/key.pem + - verify: verify_peer + - fail_if_no_peer_cert: true diff --git a/vars/ini_test.yaml b/vars/ini_test.yaml new file mode 100644 index 0000000..221a936 --- /dev/null +++ b/vars/ini_test.yaml @@ -0,0 +1,13 @@ +--- + +data: + var1: val1 + var2: val2 + section1: + aaa: asdf + bbb: 123 + ccc: 'true' + section2: + ddd: asdfasd + eee: 1234 + fff: 'false' diff --git a/vars/json_test.yaml b/vars/json_test.yaml new file mode 100644 index 0000000..f40ea73 --- /dev/null +++ b/vars/json_test.yaml @@ -0,0 +1,16 @@ +--- + +data: + string: This is a string + number: 123 + boolean: 'true' + dict: + aaa: bbb + ccc: true + list: + - eee + - fff + complex: + ggg: + - hhh + - iii: jjj diff --git a/vars/toml_test.yaml b/vars/toml_test.yaml new file mode 100644 index 0000000..623abab --- /dev/null +++ b/vars/toml_test.yaml @@ -0,0 +1,33 @@ +--- + +data: + title: TOML Example + owner: + name: Tom Preston-Werner + organization: GitHub + bio: "GitHub Cofounder & CEO\nLikes tater tots and beer." + dob: "1979-05-27T07:32:00Z" + database: + server: 192.168.1.1 + ports: [ 8001, 8001, 8002 ] + connection_max: 5000 + enabled: true + servers: + alpha: + ip: 10.0.0.1 + dc: eqdc10 + beta: + ip: 10.0.0.2 + dc: eqdc10 + country: "中国" + clients: + data: [ ["gamma", "delta"], [1, 2] ] + hosts: [ "alpha", "omega" ] + products: + - + name: Hammer + sku: 738594937 + - + name: Nail + sku: 284758393 + color: gray diff --git a/vars/xml_test.yaml b/vars/xml_test.yaml new file mode 100644 index 0000000..3fce9dd --- /dev/null +++ b/vars/xml_test.yaml @@ -0,0 +1,15 @@ +--- + +data: + root: + elem1: asd + elem2: + - asd + - zxc + with_attrs: + 'elem3 attr1="val1" attr2=val2': 123 + 'elem4 attr3="val3"': + - Value 1 + - elem5: + - Value 2 + - Value 3 diff --git a/vars/yaml_test.yaml b/vars/yaml_test.yaml new file mode 100644 index 0000000..f40ea73 --- /dev/null +++ b/vars/yaml_test.yaml @@ -0,0 +1,16 @@ +--- + +data: + string: This is a string + number: 123 + boolean: 'true' + dict: + aaa: bbb + ccc: true + list: + - eee + - fff + complex: + ggg: + - hhh + - iii: jjj diff --git a/yaml_converter.py b/yaml_converter.py new file mode 100755 index 0000000..632f71e --- /dev/null +++ b/yaml_converter.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python2 + +from jinja2 import Template +from yaml import load +import argparse +import sys + + +def parse_arguments(): + description = 'YAML to TOML converter' + + parser = argparse.ArgumentParser(description=description) + parser.add_argument( + '--format', '-f', + metavar='FORMAT', + choices=['erlang', 'ini', 'json', 'toml', 'xml', 'yaml'], + required=True, + help='Output format') + parser.add_argument( + '--path', '-p', + metavar='PATH', + default='.', + help='Path to the macros directory (default: .)') + parser.add_argument( + '--var', '-v', + metavar='NAME', + help='Read data from certain YAML variable') + parser.add_argument( + '--yaml', '-y', + metavar='FILE', + dest='yaml_fh', + type=argparse.FileType('r'), + default='-', + help='YAML file to convert (default: -)') + + return (parser.parse_args(), parser) + + +def main(): + # Parse command line arguments + (args, parser) = parse_arguments() + + # Load the YAML data to Python data structure + yaml_data = load(args.yaml_fh) + args.yaml_fh.close() + + if args.var: + yaml_data = yaml_data[args.var] + + # Read Jinja2 template as text + template_fh = open( + '%s/macros/%s_encode_macro.j2' % (args.path, args.format)) + template_text = template_fh.read() + template_fh.close() + + # Create Jinja2 template object + t = Template(template_text) + # Convert the YAML data to INI format + encode_method = getattr(t.module, '%s_encode' % args.format) + output = encode_method(yaml_data) + + # Print the result + sys.stdout.write(output.encode('utf8')) + + +if __name__ == '__main__': + main() From 92f86ba5e7d2d09c728e525bcc28c4fbfba5fc57 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Tue, 17 Feb 2015 09:54:52 +0000 Subject: [PATCH 03/32] Fixing documentation --- README.md | 222 ++++++++++++++++++++++------------ macros/erlang_encode_macro.j2 | 2 +- macros/ini_encode_macro.j2 | 2 +- macros/json_encode_macro.j2 | 2 +- macros/toml_encode_macro.j2 | 2 +- macros/xml_encode_macro.j2 | 2 +- macros/yaml_encode_macro.j2 | 2 +- 7 files changed, 149 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 2dac453..0eeac52 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ eee=1234 fff=false ``` -Look into the `site.yaml` and `templates\*.j2` for more examples. +Look into the `site.yaml` and `templates/*.j2` for more examples. Macros parameters @@ -115,133 +115,197 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where ### `erlang_encode()` - `item` - > Variable holding the input data for the macro. + + > Variable holding the input data for the macro. + - `convert_bools=true` - > Indicates whether Boolean values presented as a string should be converted - > to a real Booblean value. For example `var1: 'True'` would be represented - > as a string but using the `convert_bools=true` will convert it to a Boolean - > like it would be defined like this: `var1: true`. + + > Indicates whether Boolean values presented as a string should be converted + > to a real Booblean value. For example `var1: 'True'` would be represented + > as a string but using the `convert_bools=true` will convert it to a Boolean + > like it would be defined like this: `var1: true`. + - `convert_nums=false` - > Indicates whether number presented as a string should be converted to - > number. For example `var1: '123'` would be represented as a string but - > using the `convert_nums=true` will convert it to a number like it would - > be defined like this: `var1: 123`. + + > Indicates whether number presented as a string should be converted to + > number. For example `var1: '123'` would be represented as a string but + > using the `convert_nums=true` will convert it to a number like it would + > be defined like this: `var1: 123`. + - `first=[]` - > Indicates whether the item being processed is actually a first item. It's - > used only internally in the macro. + + > Indicates whether the item being processed is actually a first item. It's + > used only internally in the macro. + - `indent=" "` - > Defines the indentation unit. + + > Defines the indentation unit. + - `level=0` - > Indicates the initial level of the indentation. Value `0` starts indenting - > from the beginning of the line. Setting the value to higher than `0` - > indents the content by `intent * level`. + + > Indicates the initial level of the indentation. Value `0` starts indenting + > from the beginning of the line. Setting the value to higher than `0` + > indents the content by `indent * level`. ### `ini_encode()` - `item` - > Variable holding the input data for the macro. + + > Variable holding the input data for the macro. + - `delimiter="="` - > Sign separating the *property* and the *value*. By default it's set to `=` - > but it can also be set to ` = `. + + > Sign separating the *property* and the *value*. By default it's set to `'='` + > but it can also be set to `' = '`. + - `first=[]` - > Indicates whether the item being processed is actually a first item. It's - > used only internally in the macro. + + > Indicates whether the item being processed is actually a first item. It's + > used only internally in the macro. + - `quote=""` - > Allows to set the *value* quoting. Use `quote="'"` or `quote='"'`. + + > Allows to set the *value* quoting. Use `quote="'"` or `quote='"'`. + - `ucase_prop=false` - > Indicates whether the *property* should be made uppercase. + + > Indicates whether the *property* should be made uppercase. ### `json_encode()` - `item` - > Variable holding the input data for the macro. + + > Variable holding the input data for the macro. + - `convert_bools=true` - > Indicates whether Boolean values presented as a string should be converted - > to a real Booblean value. For example `var1: 'True'` would be represented - > as a string but using the `convert_bools=true` will convert it to a Boolean - > like it would be defined like this: `var1: true`. + + > Indicates whether Boolean values presented as a string should be converted + > to a real Booblean value. For example `var1: 'True'` would be represented + > as a string but using the `convert_bools=true` will convert it to a Boolean + > like it would be defined like this: `var1: true`. + - `convert_nums=false` - > Indicates whether number presented as a string should be converted to - > number. For example `var1: '123'` would be represented as a string but - > using the `convert_nums=true` will convert it to a number like it would - > be defined like this: `var1: 123`. + + > Indicates whether number presented as a string should be converted to + > number. For example `var1: '123'` would be represented as a string but + > using the `convert_nums=true` will convert it to a number like it would + > be defined like this: `var1: 123`. + - `indent=" "` - > Defines the indentation unit. + + > Defines the indentation unit. + - `level=0` - > Indicates the initial level of the indentation. Value `0` starts indenting - > from the beginning of the line. Setting the value to higher than `0` - > indents the content by `intent * level`. + + > Indicates the initial level of the indentation. Value `0` starts indenting + > from the beginning of the line. Setting the value to higher than `0` + > indents the content by `indent * level`. ### `toml_encode()` - `item` - > Variable holding the input data for the macro. + + > Variable holding the input data for the macro. + - `convert_bools=true` - > Indicates whether Boolean values presented as a string should be converted - > to a real Booblean value. For example `var1: 'True'` would be represented - > as a string but using the `convert_bools=true` will convert it to a Boolean - > like it would be defined like this: `var1: true`. + + > Indicates whether Boolean values presented as a string should be converted + > to a real Booblean value. For example `var1: 'True'` would be represented + > as a string but using the `convert_bools=true` will convert it to a Boolean + > like it would be defined like this: `var1: true`. + - `convert_nums=false` - > Indicates whether number presented as a string should be converted to - > number. For example `var1: '123'` would be represented as a string but - > using the `convert_nums=true` will convert it to a number like it would - > be defined like this: `var1: 123`. + + > Indicates whether number presented as a string should be converted to + > number. For example `var1: '123'` would be represented as a string but + > using the `convert_nums=true` will convert it to a number like it would + > be defined like this: `var1: 123`. + - `first=[]` - > Indicates whether the item being processed is actually a first item. It's - > used only internally in the macro. + + > Indicates whether the item being processed is actually a first item. It's + > used only internally in the macro. + - `indent=" "` - > Defines the indentation unit. + + > Defines the indentation unit. + - `level=0` - > Indicates the initial level of the indentation. Value `0` starts indenting - > from the beginning of the line. Setting the value to higher than `0` - > indents the content by `intent * level`. + + > Indicates the initial level of the indentation. Value `0` starts indenting + > from the beginning of the line. Setting the value to higher than `0` + > indents the content by `indent * level`. + - `prevkey=""` - > Defines the name of the previous key in the recursive macro calls. It's used - > only internally in the macro. + + > Defines the name of the previous key in the recursive macro calls. It's used + > only internally in the macro. ### `xml_encode()` - `item` - > Variable holding the input data for the macro. + + > Variable holding the input data for the macro. + - `first=[]` - > Indicates whether the item being processed is actually a first item. It's - > used only internally in the macro. + + > Indicates whether the item being processed is actually a first item. It's + > used only internally in the macro. + - `indent=" "` - > Defines the indentation unit. + + > Defines the indentation unit. + - `level=0` - > Indicates the initial level of the indentation. Value `0` starts indenting - > from the beginning of the line. Setting the value to higher than `0` - > indents the content by `intent * level`. + + > Indicates the initial level of the indentation. Value `0` starts indenting + > from the beginning of the line. Setting the value to higher than `0` + > indents the content by `indent * level`. + - `prevkey=none` - > Defines the name of the previous key in the recursive macro calls. It's used - > only internally in the macro. + + > Defines the name of the previous key in the recursive macro calls. It's used + > only internally in the macro. ### `yaml_encode()` - `item` - > Variable holding the input data for the macro. + + > Variable holding the input data for the macro. + - `convert_bools=true` - > Indicates whether Boolean values presented as a string should be converted - > to a real Booblean value. For example `var1: 'True'` would be represented - > as a string but using the `convert_bools=true` will convert it to a Boolean - > like it would be defined like this: `var1: true`. + + > Indicates whether Boolean values presented as a string should be converted + > to a real Booblean value. For example `var1: 'True'` would be represented + > as a string but using the `convert_bools=true` will convert it to a Boolean + > like it would be defined like this: `var1: true`. + - `convert_nums=false` - > Indicates whether number presented as a string should be converted to - > number. For example `var1: '123'` would be represented as a string but - > using the `convert_nums=true` will convert it to a number like it would - > be defined like this: `var1: 123`. + + > Indicates whether number presented as a string should be converted to + > number. For example `var1: '123'` would be represented as a string but + > using the `convert_nums=true` will convert it to a number like it would + > be defined like this: `var1: 123`. + - `indent=" "` - > Defines the indentation unit. + + > Defines the indentation unit. + - `level=0` - > Indicates the initial level of the indentation. Value `0` starts indenting - > from the beginning of the line. Setting the value to higher than `0` - > indents the content by `intent * level`. + + > Indicates the initial level of the indentation. Value `0` starts indenting + > from the beginning of the line. Setting the value to higher than `0` + > indents the content by `indent * level`. + - `quote='"'` - > Allows to set the *value* quoting. Use `quote="'"` or `quote='"'`. + + > Allows to set the *value* quoting. Use `quote="'"` or `quote='"'`. + - `skip_indent=false` - > Indicates whether the indentation should be skipped for the specific item. - > It's used only internally in the macro. + + > Indicates whether the indentation should be skipped for the specific item. + > It's used only internally in the macro. Limitations @@ -255,7 +319,7 @@ Boolean value (`true` and `false`). The reason for this behavior is that if you want to pass a Boolean variable in Ansible, you have to actually quote it (e.g. `var1: "{{ my_boolean_var }}"`). Once the variable is quoted, it's not a Boolean variable any more. If you want to use Boolean as a string in your final -configuration file, try to use value with a space (e.g. `' true'`). +configuration file, try to use the value with a space (e.g. `' true'`). ### XML macro diff --git a/macros/erlang_encode_macro.j2 b/macros/erlang_encode_macro.j2 index 022050f..1fa247b 100644 --- a/macros/erlang_encode_macro.j2 +++ b/macros/erlang_encode_macro.j2 @@ -89,4 +89,4 @@ {%- endif %} {%- endif %} -{%- endmacro -%} +{%- endmacro %} diff --git a/macros/ini_encode_macro.j2 b/macros/ini_encode_macro.j2 index dcc5a39..7855a95 100644 --- a/macros/ini_encode_macro.j2 +++ b/macros/ini_encode_macro.j2 @@ -33,4 +33,4 @@ {%- endfor %} {%- endif %} {%- endfor %} -{%- endmacro -%} +{%- endmacro %} diff --git a/macros/json_encode_macro.j2 b/macros/json_encode_macro.j2 index f44d577..b2be618 100644 --- a/macros/json_encode_macro.j2 +++ b/macros/json_encode_macro.j2 @@ -88,4 +88,4 @@ {%- endif %} {%- endif %} -{%- endmacro -%} +{%- endmacro %} diff --git a/macros/toml_encode_macro.j2 b/macros/toml_encode_macro.j2 index 61e6a31..85b66d8 100644 --- a/macros/toml_encode_macro.j2 +++ b/macros/toml_encode_macro.j2 @@ -113,4 +113,4 @@ {{ item }}{{ "\n" }} {%- endif %} {%- endif %} -{%- endmacro -%} +{%- endmacro %} diff --git a/macros/xml_encode_macro.j2 b/macros/xml_encode_macro.j2 index 15e7d9b..fad5fcb 100644 --- a/macros/xml_encode_macro.j2 +++ b/macros/xml_encode_macro.j2 @@ -75,4 +75,4 @@ {{ "\n" }} {%- endfor %} {%- endif %} -{%- endmacro -%} +{%- endmacro %} diff --git a/macros/yaml_encode_macro.j2 b/macros/yaml_encode_macro.j2 index 39c8561..fb55f33 100644 --- a/macros/yaml_encode_macro.j2 +++ b/macros/yaml_encode_macro.j2 @@ -76,4 +76,4 @@ skip_indent=true) }} {%- endfor %} {%- endif %} -{%- endmacro -%} +{%- endmacro %} From caa3470531379e4ffa45ed272889854c6468fd65 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Tue, 17 Feb 2015 10:16:01 +0000 Subject: [PATCH 04/32] Fixing boolean values for type sensitive formats and adding new lines in Erlang and JSON output --- TODO.md | 7 ------- macros/erlang_encode_macro.j2 | 6 +++++- macros/json_encode_macro.j2 | 6 +++++- macros/toml_encode_macro.j2 | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) delete mode 100644 TODO.md diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 627f68b..0000000 --- a/TODO.md +++ /dev/null @@ -1,7 +0,0 @@ -TODO -==== - -- Fix formating when passing real Boolean value (True -> true) in all - Boolean-sensitive formats. Probably requires a new section. -- Missing new line at the end of Erlang and JSON file generated by the Python - script diff --git a/macros/erlang_encode_macro.j2 b/macros/erlang_encode_macro.j2 index 1fa247b..46838f3 100644 --- a/macros/erlang_encode_macro.j2 +++ b/macros/erlang_encode_macro.j2 @@ -36,6 +36,10 @@ {%- if not loop.last -%} {{ ",\n" }} + {%- endif -%} + + {%- if level == 0 -%} + {{ "\n" }} {%- endif %} {%- endfor %} @@ -50,7 +54,7 @@ {#- Item is a value of a number, null or boolean -#} - {{ item }} + {{ item | lower }} {%- elif item is string %} {#- The item is a string -#} diff --git a/macros/json_encode_macro.j2 b/macros/json_encode_macro.j2 index b2be618..e2fbbb8 100644 --- a/macros/json_encode_macro.j2 +++ b/macros/json_encode_macro.j2 @@ -39,6 +39,10 @@ {{ "}" }} {%- endif %} + {%- if level == 0 -%} + {{ "\n" }} + {%- endif %} + {%- elif item == "null" or item is number or (convert_nums and ( item | int | string == item or @@ -50,7 +54,7 @@ {#- Item is a value of a number, null or boolean -#} - {{ item }} + {{ item | lower }} {%- elif item is string %} {#- The item is a string -#} diff --git a/macros/toml_encode_macro.j2 b/macros/toml_encode_macro.j2 index 85b66d8..82e678f 100644 --- a/macros/toml_encode_macro.j2 +++ b/macros/toml_encode_macro.j2 @@ -91,7 +91,7 @@ item == "False" %} {#- The item is a number or boolean -#} - {{ item }}{{ "\n" }} + {{ item | lower }}{{ "\n" }} {%- elif item is string %} {#- The item is a string -#} From 36a782671fe6785a8d4e97acb0addd48b0be6653 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Tue, 17 Feb 2015 12:50:09 +0000 Subject: [PATCH 05/32] Improving documentation --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0eeac52..a183849 100644 --- a/README.md +++ b/README.md @@ -313,13 +313,13 @@ Limitations ### Erlang, JSON, TOML and YAML macros -In these sensitive file formats, the Boolean values represented as string +In these type-sensitive file formats, the Boolean values represented as string (`"true"`, `"True"`, `"false"` and `"False"`) are automatically converted to Boolean value (`true` and `false`). The reason for this behavior is that if you want to pass a Boolean variable in Ansible, you have to actually quote it (e.g. -`var1: "{{ my_boolean_var }}"`). Once the variable is quoted, it's not a Boolean -variable any more. If you want to use Boolean as a string in your final -configuration file, try to use the value with a space (e.g. `' true'`). +`var1: "{{ my_boolean_var }}"`). Once the variable is quoted, it's not Boolean +any more. If you want to use Boolean as a string in your final configuration +file, try to use the value with a space (e.g. `' true'`). ### XML macro From 571adb3751d3580ce25e46412f0572ada605cd39 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Mon, 23 Feb 2015 12:06:50 +0000 Subject: [PATCH 06/32] Adding example for properties of the same name --- vars/ini_test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vars/ini_test.yaml b/vars/ini_test.yaml index 221a936..0314a0e 100644 --- a/vars/ini_test.yaml +++ b/vars/ini_test.yaml @@ -4,7 +4,9 @@ data: var1: val1 var2: val2 section1: - aaa: asdf + aaa: + - asdf + - zxcv bbb: 123 ccc: 'true' section2: From 6bd9035264cbe215dec3bfb12448125e4a5ecb16 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Mon, 23 Feb 2015 12:08:10 +0000 Subject: [PATCH 07/32] Adding simplified INI format with sections as comment --- macros/ini_encode_macro.j2 | 26 ++++++++++++++++++++++---- site.yaml | 12 ++++++++++++ templates/test.ini_simple.j2 | 9 +++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 templates/test.ini_simple.j2 diff --git a/macros/ini_encode_macro.j2 b/macros/ini_encode_macro.j2 index 7855a95..0f877e1 100644 --- a/macros/ini_encode_macro.j2 +++ b/macros/ini_encode_macro.j2 @@ -7,6 +7,7 @@ delimiter="=", first=[], quote="", + section_is_comment=false, ucase_prop=false) %} {#- First process vars without a section #} @@ -15,7 +16,14 @@ {%- if first | length == 0 -%} {%- if first.append(1) %}{% endif %} {%- endif -%} - {{ (property | upper if ucase_prop else property) ~ delimiter ~ quote ~ value ~ quote ~ "\n" }} + + {%- if value is string or value is number -%} + {{ (property | upper if ucase_prop else property) ~ delimiter ~ quote ~ value ~ quote ~ "\n" }} + {%- else %} + {%- for item in value -%} + {{ (property | upper if ucase_prop else property) ~ delimiter ~ quote ~ item ~ quote ~ "\n" }} + {%- endfor %} + {%- endif %} {%- endif %} {%- endfor %} @@ -26,10 +34,20 @@ {%- if first.append(1) %}{% endif %} {%- else -%} {{ "\n" }} - {%- endif -%} - {{ "[" ~ section ~ "]\n" }} + {%- endif %} + {%- if section_is_comment -%} + {{ "# " ~ section ~ "\n" }} + {%- else -%} + {{ "[" ~ section ~ "]\n" }} + {%- endif %} {%- for property, value in options.iteritems() | sort -%} - {{ (property | upper if ucase_prop else property) ~ delimiter ~ quote ~ value ~ quote ~ "\n" }} + {%- if value is string or value is number -%} + {{ (property | upper if ucase_prop else property) ~ delimiter ~ quote ~ value ~ quote ~ "\n" }} + {%- else %} + {%- for item in value -%} + {{ (property | upper if ucase_prop else property) ~ delimiter ~ quote ~ item ~ quote ~ "\n" }} + {%- endfor %} + {%- endif %} {%- endfor %} {%- endif %} {%- endfor %} diff --git a/site.yaml b/site.yaml index d85705f..e416feb 100644 --- a/site.yaml +++ b/site.yaml @@ -24,6 +24,18 @@ tags: - ini +- name: Simplified INI encode example + hosts: all + vars_files: + - vars/ini_test.yaml + tasks: + - name: Create test.simple file + template: + src: templates/test.ini_simple.j2 + dest: /tmp/test.ini_simple + tags: + - simple + - name: JSON encode example hosts: all vars_files: diff --git a/templates/test.ini_simple.j2 b/templates/test.ini_simple.j2 new file mode 100644 index 0000000..648409c --- /dev/null +++ b/templates/test.ini_simple.j2 @@ -0,0 +1,9 @@ +# +# This file is managed by Ansible. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +{% from "macros/ini_encode_macro.j2" import ini_encode with context -%} + +{{ ini_encode(data, section_is_comment=true, delimiter=" ") }} From ade65ef57c22fe9cf8c339be21c3bcb93f6d25da Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Mon, 23 Feb 2015 12:10:04 +0000 Subject: [PATCH 08/32] Changing single to double quotes --- README.md | 2 +- templates/test.erlang.j2 | 2 +- templates/test.ini.j2 | 2 +- templates/test.json.j2 | 2 +- templates/test.toml.j2 | 2 +- templates/test.xml.j2 | 2 +- templates/test.yaml.j2 | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a183849..6992b88 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Where the `templates/test.ini.j2` contains: # Any changes will be automatically reverted. # -{% from 'macros/ini_encode_macro.j2' import ini_encode with context -%} +{% from "macros/ini_encode_macro.j2" import ini_encode with context -%} {{ ini_encode(data) }} ``` diff --git a/templates/test.erlang.j2 b/templates/test.erlang.j2 index 8f17d2a..0f3ae12 100644 --- a/templates/test.erlang.j2 +++ b/templates/test.erlang.j2 @@ -4,6 +4,6 @@ %% Any changes will be automatically reverted. %% -{% from 'macros/erlang_encode_macro.j2' import erlang_encode with context -%} +{% from "macros/erlang_encode_macro.j2" import erlang_encode with context -%} {{ erlang_encode(data, convert_nums=true) }} diff --git a/templates/test.ini.j2 b/templates/test.ini.j2 index d1d0363..f5475b7 100644 --- a/templates/test.ini.j2 +++ b/templates/test.ini.j2 @@ -4,6 +4,6 @@ # Any changes will be automatically reverted. # -{% from 'macros/ini_encode_macro.j2' import ini_encode with context -%} +{% from "macros/ini_encode_macro.j2" import ini_encode with context -%} {{ ini_encode(data) }} diff --git a/templates/test.json.j2 b/templates/test.json.j2 index 159b0b2..25b008d 100644 --- a/templates/test.json.j2 +++ b/templates/test.json.j2 @@ -1,3 +1,3 @@ -{% from 'macros/json_encode_macro.j2' import json_encode with context -%} +{% from "macros/json_encode_macro.j2" import json_encode with context -%} {{ json_encode(data, convert_nums=true) }} diff --git a/templates/test.toml.j2 b/templates/test.toml.j2 index 5aa3d29..6a0d73e 100644 --- a/templates/test.toml.j2 +++ b/templates/test.toml.j2 @@ -4,6 +4,6 @@ # Any changes will be automatically reverted. # -{% from 'macros/toml_encode_macro.j2' import toml_encode with context -%} +{% from "macros/toml_encode_macro.j2" import toml_encode with context -%} {{ toml_encode(data) }} diff --git a/templates/test.xml.j2 b/templates/test.xml.j2 index d63814d..6d21b7d 100644 --- a/templates/test.xml.j2 +++ b/templates/test.xml.j2 @@ -8,6 +8,6 @@ - --> -{% from 'macros/xml_encode_macro.j2' import xml_encode with context -%} +{% from "macros/xml_encode_macro.j2" import xml_encode with context -%} {{ xml_encode(data) }} diff --git a/templates/test.yaml.j2 b/templates/test.yaml.j2 index cddd19c..4c736a8 100644 --- a/templates/test.yaml.j2 +++ b/templates/test.yaml.j2 @@ -4,6 +4,6 @@ # Any changes will be automatically reverted. # -{% from 'macros/yaml_encode_macro.j2' import yaml_encode with context -%} +{% from "macros/yaml_encode_macro.j2" import yaml_encode with context -%} {{ yaml_encode(data, convert_nums=true) }} From 55e80159c7c50a288ecdc2929bd065ec6a253810 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Thu, 26 Feb 2015 15:20:06 +0000 Subject: [PATCH 09/32] Adding missing dot at the EOF --- macros/erlang_encode_macro.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/erlang_encode_macro.j2 b/macros/erlang_encode_macro.j2 index 46838f3..af1c3ae 100644 --- a/macros/erlang_encode_macro.j2 +++ b/macros/erlang_encode_macro.j2 @@ -39,7 +39,7 @@ {%- endif -%} {%- if level == 0 -%} - {{ "\n" }} + {{ ".\n" }} {%- endif %} {%- endfor %} From 0e787343c4061a699ef7fecd29dc6b40758a26db Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Thu, 26 Feb 2015 15:48:30 +0000 Subject: [PATCH 10/32] Erlang config always starts by list --- macros/erlang_encode_macro.j2 | 14 ++++++-------- vars/erlang_test.yaml | 10 +++++----- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/macros/erlang_encode_macro.j2 b/macros/erlang_encode_macro.j2 index af1c3ae..f7c83ee 100644 --- a/macros/erlang_encode_macro.j2 +++ b/macros/erlang_encode_macro.j2 @@ -11,11 +11,9 @@ level=0) %} {%- if item is mapping %} - {#- The item is a dict #} + {#- The item is a dict -#} - {%- if first | length > 0 and item | length > 0 -%} - {{ "\n" }} - {%- endif %} + {{ "\n" }} {%- if first.append(1) %}{% endif %} @@ -37,10 +35,6 @@ {%- if not loop.last -%} {{ ",\n" }} {%- endif -%} - - {%- if level == 0 -%} - {{ ".\n" }} - {%- endif %} {%- endfor %} {%- elif item == "null" or item is number or @@ -92,5 +86,9 @@ {{ "]" }} {%- endif %} + {%- if level == 0 -%} + {{ ".\n" }} + {%- endif %} + {%- endif %} {%- endmacro %} diff --git a/vars/erlang_test.yaml b/vars/erlang_test.yaml index f0980dc..268f54e 100644 --- a/vars/erlang_test.yaml +++ b/vars/erlang_test.yaml @@ -1,12 +1,12 @@ --- data: - rabbit: - tcp_listeners: - - 127.0.0.1: 5672 - ssl_listeners: + - rabbit: + - tcp_listeners: + - '"127.0.0.1"': 5672 + - ssl_listeners: - 5671 - ssl_options: + - ssl_options: - cacertfile: /path/to/testca/cacert.pem - certfile: /path/to/server/cert.pem - keyfile: /path/to/server/key.pem From 1a84d42c4014c2f765e27e34db37fd3a1706945e Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Fri, 6 Mar 2015 14:00:51 +0000 Subject: [PATCH 11/32] Adding description of section_is_comment param in ini_encode macro --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 6992b88..3a717ab 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,10 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > Allows to set the *value* quoting. Use `quote="'"` or `quote='"'`. +- `section_is_comment=false` + + > Prints sections as comments. + - `ucase_prop=false` > Indicates whether the *property* should be made uppercase. From 85b54960481c0ca2c56bc0cba14558f8dd64e384 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Mon, 9 Mar 2015 12:45:45 +0000 Subject: [PATCH 12/32] Making boolean conversion optional --- README.md | 35 +++++++++++++++++++++++++---------- macros/erlang_encode_macro.j2 | 7 ++----- macros/json_encode_macro.j2 | 7 ++----- macros/toml_encode_macro.j2 | 7 ++----- macros/yaml_encode_macro.j2 | 7 ++----- 5 files changed, 33 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 3a717ab..afef231 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > Variable holding the input data for the macro. -- `convert_bools=true` +- `convert_bools=false` > Indicates whether Boolean values presented as a string should be converted > to a real Booblean value. For example `var1: 'True'` would be represented @@ -130,7 +130,8 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > Indicates whether number presented as a string should be converted to > number. For example `var1: '123'` would be represented as a string but > using the `convert_nums=true` will convert it to a number like it would - > be defined like this: `var1: 123`. + > be defined like this: `var1: 123`. You can also use the YAML type casting + > to convert string to number (e.g. `!!int "1234"`, `!!float "3.14"`). - `first=[]` @@ -181,7 +182,7 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > Variable holding the input data for the macro. -- `convert_bools=true` +- `convert_bools=false` > Indicates whether Boolean values presented as a string should be converted > to a real Booblean value. For example `var1: 'True'` would be represented @@ -193,7 +194,8 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > Indicates whether number presented as a string should be converted to > number. For example `var1: '123'` would be represented as a string but > using the `convert_nums=true` will convert it to a number like it would - > be defined like this: `var1: 123`. + > be defined like this: `var1: 123`. You can also use the YAML type casting + > to convert string to number (e.g. `!!int "1234"`, `!!float "3.14"`). - `indent=" "` @@ -212,19 +214,22 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > Variable holding the input data for the macro. -- `convert_bools=true` +- `convert_bools=false` > Indicates whether Boolean values presented as a string should be converted > to a real Booblean value. For example `var1: 'True'` would be represented > as a string but using the `convert_bools=true` will convert it to a Boolean - > like it would be defined like this: `var1: true`. + > like it would be defined like this: `var1: true`. You can also use the YAML + > type casting by using `!!bool` in front of your value (e.g. + > `!!bool "true"`). - `convert_nums=false` > Indicates whether number presented as a string should be converted to > number. For example `var1: '123'` would be represented as a string but > using the `convert_nums=true` will convert it to a number like it would - > be defined like this: `var1: 123`. + > be defined like this: `var1: 123`. You can also use the YAML type casting + > to convert string to number (e.g. `!!int "1234"`, `!!float "3.14"`). - `first=[]` @@ -278,7 +283,7 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > Variable holding the input data for the macro. -- `convert_bools=true` +- `convert_bools=false` > Indicates whether Boolean values presented as a string should be converted > to a real Booblean value. For example `var1: 'True'` would be represented @@ -290,7 +295,8 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > Indicates whether number presented as a string should be converted to > number. For example `var1: '123'` would be represented as a string but > using the `convert_nums=true` will convert it to a number like it would - > be defined like this: `var1: 123`. + > be defined like this: `var1: 123`. You can also use the YAML type casting + > to convert string to number (e.g. `!!int "1234"`, `!!float "3.14"`). - `indent=" "` @@ -326,6 +332,13 @@ any more. If you want to use Boolean as a string in your final configuration file, try to use the value with a space (e.g. `' true'`). +### INI macro + +This macro doesn't respect the order in which the data was defined. It probably +need to be fixed as some config files might allow to overwrite previous +definitions. + + ### XML macro YAML doesn't allow to fully map XML data with attributes. This is why XML @@ -352,7 +365,9 @@ Will produce the following XML file (`elem` has the same set of attributes): ``` Another limitation of the XML macro is that attributes can not have value -derivated from a Jinja2 variable in the Ansible context. +derivated from a Jinja2 variable in the Ansible context. This macro also doesn't +allow to set order of the elements so it's suitable only for order-insensitive +XML files. Python example diff --git a/macros/erlang_encode_macro.j2 b/macros/erlang_encode_macro.j2 index f7c83ee..2b0639f 100644 --- a/macros/erlang_encode_macro.j2 +++ b/macros/erlang_encode_macro.j2 @@ -4,7 +4,7 @@ {%- macro erlang_encode( item, - convert_bools=true, + convert_bools=false, convert_nums=false, first=[], indent=" ", @@ -41,10 +41,7 @@ (convert_nums and ( item | int | string == item or item | float | string == item)) or - item == "true" or - item == "True" or - item == "false" or - item == "False" %} + item in ["true", "True", "false", "False"] %} {#- Item is a value of a number, null or boolean -#} diff --git a/macros/json_encode_macro.j2 b/macros/json_encode_macro.j2 index e2fbbb8..2513466 100644 --- a/macros/json_encode_macro.j2 +++ b/macros/json_encode_macro.j2 @@ -4,7 +4,7 @@ {%- macro json_encode( item, - convert_bools=true, + convert_bools=false, convert_nums=false, indent=" ", level=0) %} @@ -47,10 +47,7 @@ (convert_nums and ( item | int | string == item or item | float | string == item)) or - item == "true" or - item == "True" or - item == "false" or - item == "False" %} + item in ["true", "True", "false", "False"] %} {#- Item is a value of a number, null or boolean -#} diff --git a/macros/toml_encode_macro.j2 b/macros/toml_encode_macro.j2 index 82e678f..e4c801d 100644 --- a/macros/toml_encode_macro.j2 +++ b/macros/toml_encode_macro.j2 @@ -4,7 +4,7 @@ {%- macro toml_encode( item, - convert_bools=true, + convert_bools=false, convert_nums=false, first=[], indent=" ", @@ -85,10 +85,7 @@ (convert_nums and ( item | int | string == item or item | float | string == item)) or - item == "true" or - item == "True" or - item == "false" or - item == "False" %} + item in ["true", "True", "false", "False"] %} {#- The item is a number or boolean -#} {{ item | lower }}{{ "\n" }} diff --git a/macros/yaml_encode_macro.j2 b/macros/yaml_encode_macro.j2 index fb55f33..161f62b 100644 --- a/macros/yaml_encode_macro.j2 +++ b/macros/yaml_encode_macro.j2 @@ -4,7 +4,7 @@ {%- macro yaml_encode( item, - convert_bools=true, + convert_bools=false, convert_nums=false, indent=" ", level=0, @@ -42,10 +42,7 @@ item | int | string == item or item | float | string == item)) or (convert_bools and ( - item == "true" or - item == "True" or - item == "false" or - item == "False")) %} + item in ["true", "True", "false", "False"])) %} {#- The item is a number or boolean -#} {{ " " ~ item | lower ~ "\n" }} From 6abd345202d50bf886c979ad535962cbb251cf77 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Mon, 9 Mar 2015 17:20:49 +0000 Subject: [PATCH 13/32] Adding support for Apache format --- README.md | 40 ++++++++++++- macros/apache_encode_macro.j2 | 105 ++++++++++++++++++++++++++++++++++ site.yaml | 12 ++++ templates/test.apache.j2 | 9 +++ vars/apache_test.yaml | 64 +++++++++++++++++++++ yaml_converter.py | 2 +- 6 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 macros/apache_encode_macro.j2 create mode 100644 templates/test.apache.j2 create mode 100644 vars/apache_test.yaml diff --git a/README.md b/README.md index afef231..01ac03c 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ convert the Python data structure to another format. Supported formats ----------------- -- Erlang config +- Apache config format +- Erlang config format - INI - JSON - TOML @@ -112,6 +113,43 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where `indent` is the keyword). +### `apache_encode()` + +- `item` + + > Variable holding the input data for the macro. + +- `convert_bools=false` + + > Indicates whether Boolean values presented as a string should be converted + > to a real Booblean value. For example `var1: 'True'` would be represented + > as a string but using the `convert_bools=true` will convert it to a Boolean + > like it would be defined like this: `var1: true`. + +- `convert_nums=false` + + > Indicates whether number presented as a string should be converted to + > number. For example `var1: '123'` would be represented as a string but + > using the `convert_nums=true` will convert it to a number like it would + > be defined like this: `var1: 123`. You can also use the YAML type casting + > to convert string to number (e.g. `!!int "1234"`, `!!float "3.14"`). + +- `indent=" "` + + > Defines the indentation unit. + +- `type="section"` + + > Defines the type of the `item` in the recursive macro calls. It's used only + > internally in the macro. + +- `level=0` + + > Indicates the initial level of the indentation. Value `0` starts indenting + > from the beginning of the line. Setting the value to higher than `0` + > indents the content by `indent * level`. + + ### `erlang_encode()` - `item` diff --git a/macros/apache_encode_macro.j2 b/macros/apache_encode_macro.j2 new file mode 100644 index 0000000..a3f833c --- /dev/null +++ b/macros/apache_encode_macro.j2 @@ -0,0 +1,105 @@ +{# + # Jinja2 macro which converts Python data structure to Apache format + #} + +{%- macro apache_encode( + item, + convert_bools=false, + convert_nums=false, + indent=" ", + level=0, + type="sections") %} + + {%- if type == "sections" %} + {%- for c in item["content"] %} + {#- First check if this section has options #} + {%- if "options" in c -%} + {{ apache_encode( + c["options"], + convert_bools=convert_bools, + convert_nums=convert_nums, + type="options", + level=level+1) }} + {%- endif %} + + {#- Check if this section has some sub-sections #} + {%- if "sections" in c %} + + {%- for s in c["sections"] -%} + {{ indent * level ~ "<" ~ s["name"] ~ (" " ~ + apache_encode( + s["param"], + convert_bools=convert_bools, + convert_nums=convert_nums, + type="value", + level=level+1) + if s["param"] else "") ~ ">\n" ~ + apache_encode( + s, + convert_bools=convert_bools, + convert_nums=convert_nums, + type="sections", + level=level+1) ~ + indent * level ~ "\n" }} + + {%- if not loop.last -%} + {{ "\n" }} + {%- endif %} + {%- endfor %} + {%- endif %} + + {%- if not loop.last -%} + {{ "\n" }} + {%- endif %} + {%- endfor %} + + {%- elif type == "options" %} + {%- for o in item -%} + {%- for key, val in o.iteritems() -%} + {{ indent * (level-1) ~ key ~ " " ~ + apache_encode( + val, + convert_bools=convert_bools, + convert_nums=convert_nums, + type="value", + level=level+1) ~ "\n" }} + {%- endfor %} + {%- endfor %} + + {%- elif type == "value" %} + {%- if item == True or item == False or + convert_bools and (item in ["true", "True", "false", "False"]) %} + {#- Value is a boolean -#} + {{ item | lower }} + + {%- elif item is number %} + {#- Value is a number -#} + {{ item }} + + {%- elif item is string %} + {#- Value is a string #} + {%- if " " in item or "\t" in item or "\n" in item -%} + "{{ item | replace('"', '\\"') }}" + {%- else -%} + {{ item }} + {%- endif %} + + {%- elif item is not mapping %} + {#- Value is a list #} + {%- for v in item -%} + {{ apache_encode( + v, + convert_bools=convert_bools, + convert_nums=convert_nums, + type="value", + level=level+1) }} + + {%- if not loop.last -%} + {{ " " }} + {%- endif %} + {%- endfor %} + {%- endif %} + + {%- endif %} + +{%- endmacro %} diff --git a/site.yaml b/site.yaml index e416feb..be37447 100644 --- a/site.yaml +++ b/site.yaml @@ -1,5 +1,17 @@ --- +- name: Apache encode example + hosts: all + vars_files: + - vars/apache_test.yaml + tasks: + - name: Create test.apache file + template: + src: templates/test.apache.j2 + dest: /tmp/test.apache + tags: + - apache + - name: Erlang encode example hosts: all vars_files: diff --git a/templates/test.apache.j2 b/templates/test.apache.j2 new file mode 100644 index 0000000..1306411 --- /dev/null +++ b/templates/test.apache.j2 @@ -0,0 +1,9 @@ +# +# This file is managed by Ansible. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +{% from "macros/apache_encode_macro.j2" import apache_encode with context -%} + +{{ apache_encode(data, convert_nums=true) }} diff --git a/vars/apache_test.yaml b/vars/apache_test.yaml new file mode 100644 index 0000000..26095c7 --- /dev/null +++ b/vars/apache_test.yaml @@ -0,0 +1,64 @@ +--- + +data: + content: + - options: + - ServerName: localhost + - Listen: 8080 + - PidFile: tmp/httpd.pid + - LockFile: tmp/accept.lock + - options: + - "# This is an example of a comment": "" + - LoadModule: + - authz_host_module + - /usr/libexec/apache2/mod_authz_host.so + - LoadModule: + - dir_module + - /usr/libexec/apache2/mod_dir.so + - LoadModule: + - env_module + - /usr/libexec/apache2/mod_env.so + - LoadModule: + - mime_module + - /usr/libexec/apache2/mod_mime.so + - LoadModule: + - log_config_module + - /usr/libexec/apache2/mod_log_config.so + - LoadModule: + - rewrite_module + - /usr/libexec/apache2/mod_rewrite.so + - LoadModule: + - php5_module + - /usr/local/opt/php53/libexec/apache2/libphp5.so + - LogLevel: info + - options: + - ErrorLog: "|cat" + - LogFormat: + - "%h %l %u %t \"%r\" %>s %b" + - common + - CustomLog: + - "|cat" + - common + - options: + - DocumentRoot: build + - sections: + - name: Directory + param: build + content: + - options: + - AllowOverride: all + - Order: allow,deny + - Allow: + - from + - all + - options: + - AddType: + - application/x-httpd-php + - .php + - DirectoryIndex: + - index.html + - index.php + - options: + - SetEnv: + - LOCAL_SERVER + - true diff --git a/yaml_converter.py b/yaml_converter.py index 632f71e..3c1e9e0 100755 --- a/yaml_converter.py +++ b/yaml_converter.py @@ -13,7 +13,7 @@ def parse_arguments(): parser.add_argument( '--format', '-f', metavar='FORMAT', - choices=['erlang', 'ini', 'json', 'toml', 'xml', 'yaml'], + choices=['apache', 'erlang', 'ini', 'json', 'toml', 'xml', 'yaml'], required=True, help='Output format') parser.add_argument( From 80157bc7da044d4041a154a797d7b4f975a2cdaf Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Mon, 9 Mar 2015 17:20:49 +0000 Subject: [PATCH 14/32] Removing convert params from templates --- templates/test.apache.j2 | 2 +- templates/test.erlang.j2 | 2 +- templates/test.json.j2 | 2 +- templates/test.yaml.j2 | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/test.apache.j2 b/templates/test.apache.j2 index 1306411..43fdb29 100644 --- a/templates/test.apache.j2 +++ b/templates/test.apache.j2 @@ -6,4 +6,4 @@ {% from "macros/apache_encode_macro.j2" import apache_encode with context -%} -{{ apache_encode(data, convert_nums=true) }} +{{ apache_encode(data) }} diff --git a/templates/test.erlang.j2 b/templates/test.erlang.j2 index 0f3ae12..6088e8f 100644 --- a/templates/test.erlang.j2 +++ b/templates/test.erlang.j2 @@ -6,4 +6,4 @@ {% from "macros/erlang_encode_macro.j2" import erlang_encode with context -%} -{{ erlang_encode(data, convert_nums=true) }} +{{ erlang_encode(data) }} diff --git a/templates/test.json.j2 b/templates/test.json.j2 index 25b008d..ddd802a 100644 --- a/templates/test.json.j2 +++ b/templates/test.json.j2 @@ -1,3 +1,3 @@ {% from "macros/json_encode_macro.j2" import json_encode with context -%} -{{ json_encode(data, convert_nums=true) }} +{{ json_encode(data) }} diff --git a/templates/test.yaml.j2 b/templates/test.yaml.j2 index 4c736a8..656539c 100644 --- a/templates/test.yaml.j2 +++ b/templates/test.yaml.j2 @@ -6,4 +6,4 @@ {% from "macros/yaml_encode_macro.j2" import yaml_encode with context -%} -{{ yaml_encode(data, convert_nums=true) }} +{{ yaml_encode(data) }} From 8eccf4d328b88d5332214794ef01025010edc4ea Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Tue, 10 Mar 2015 15:22:47 +0000 Subject: [PATCH 15/32] Fixing dict key check in Apache macro --- macros/apache_encode_macro.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/apache_encode_macro.j2 b/macros/apache_encode_macro.j2 index a3f833c..455135d 100644 --- a/macros/apache_encode_macro.j2 +++ b/macros/apache_encode_macro.j2 @@ -33,7 +33,7 @@ convert_nums=convert_nums, type="value", level=level+1) - if s["param"] else "") ~ ">\n" ~ + if "param" in s else "") ~ ">\n" ~ apache_encode( s, convert_bools=convert_bools, From aea9e1995e9f3f885971ba10cd61e810e1456957 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Tue, 10 Mar 2015 17:01:06 +0000 Subject: [PATCH 16/32] Fixing formating of the Apache macro --- macros/apache_encode_macro.j2 | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/macros/apache_encode_macro.j2 b/macros/apache_encode_macro.j2 index 455135d..91d633e 100644 --- a/macros/apache_encode_macro.j2 +++ b/macros/apache_encode_macro.j2 @@ -24,7 +24,6 @@ {#- Check if this section has some sub-sections #} {%- if "sections" in c %} - {%- for s in c["sections"] -%} {{ indent * level ~ "<" ~ s["name"] ~ (" " ~ apache_encode( @@ -42,13 +41,18 @@ level=level+1) ~ indent * level ~ "\n" }} - {%- if not loop.last -%} + {%- if not loop.last -%} {{ "\n" }} {%- endif %} {%- endfor %} {%- endif %} - {%- if not loop.last -%} + {%- if not loop.last and + ( + "options" in c and c["options"] | length > 0 or + "sections" in c and c["sections"] | length > 0 + ) + -%} {{ "\n" }} {%- endif %} {%- endfor %} From 96f09380bc5f9749125c56c35415d584aed60423 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Wed, 11 Mar 2015 15:35:18 +0000 Subject: [PATCH 17/32] Prevent ot create empty sections in the Apache macro --- macros/apache_encode_macro.j2 | 47 +++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/macros/apache_encode_macro.j2 b/macros/apache_encode_macro.j2 index 91d633e..0e5bcb7 100644 --- a/macros/apache_encode_macro.j2 +++ b/macros/apache_encode_macro.j2 @@ -24,25 +24,34 @@ {#- Check if this section has some sub-sections #} {%- if "sections" in c %} - {%- for s in c["sections"] -%} - {{ indent * level ~ "<" ~ s["name"] ~ (" " ~ - apache_encode( - s["param"], - convert_bools=convert_bools, - convert_nums=convert_nums, - type="value", - level=level+1) - if "param" in s else "") ~ ">\n" ~ - apache_encode( - s, - convert_bools=convert_bools, - convert_nums=convert_nums, - type="sections", - level=level+1) ~ - indent * level ~ "\n" }} + {%- for s in c["sections"] %} + {%- if ( + "options" in s["content"][0] and + s["content"][0]["options"] | length > 0 + ) or + ("sections" in s["content"][0] and + s["content"][0]["sections"] | length > 0 + ) -%} - {%- if not loop.last -%} - {{ "\n" }} + {{ indent * level ~ "<" ~ s["name"] ~ (" " ~ + apache_encode( + s["param"], + convert_bools=convert_bools, + convert_nums=convert_nums, + type="value", + level=level+1) + if "param" in s else "") ~ ">\n" ~ + apache_encode( + s, + convert_bools=convert_bools, + convert_nums=convert_nums, + type="sections", + level=level+1) ~ + indent * level ~ "\n" }} + + {%- if not loop.last -%} + {{ "\n" }} + {%- endif %} {%- endif %} {%- endfor %} {%- endif %} @@ -82,7 +91,7 @@ {%- elif item is string %} {#- Value is a string #} - {%- if " " in item or "\t" in item or "\n" in item -%} + {%- if " " in item or "\t" in item or "\n" in item or item == "" -%} "{{ item | replace('"', '\\"') }}" {%- else -%} {{ item }} From 3175dc725061c0b6666db48e0b08727e7ffce680 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Thu, 12 Mar 2015 00:42:32 +0000 Subject: [PATCH 18/32] Fixing the check of empty sub-sections in the Apache macro --- macros/apache_encode_macro.j2 | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/macros/apache_encode_macro.j2 b/macros/apache_encode_macro.j2 index 0e5bcb7..7ae13f9 100644 --- a/macros/apache_encode_macro.j2 +++ b/macros/apache_encode_macro.j2 @@ -22,17 +22,20 @@ level=level+1) }} {%- endif %} + {%- set is_empty = [] %} + {#- Check if this section has some sub-sections #} {%- if "sections" in c %} {%- for s in c["sections"] %} - {%- if ( - "options" in s["content"][0] and - s["content"][0]["options"] | length > 0 - ) or - ("sections" in s["content"][0] and - s["content"][0]["sections"] | length > 0 - ) -%} + {#- Check for empty sub-sections #} + {%- for i in s["content"] %} + {%- if ('options' in i and i['options'] | length > 0) or ('sections' in i and i['sections'] | length > 0) %} + {%- if is_empty.append(1) %}{% endif %} + {%- endif %} + {%- endfor %} + + {%- if is_empty | length > 0 -%} {{ indent * level ~ "<" ~ s["name"] ~ (" " ~ apache_encode( s["param"], @@ -59,7 +62,10 @@ {%- if not loop.last and ( "options" in c and c["options"] | length > 0 or - "sections" in c and c["sections"] | length > 0 + ( + "sections" in c and c["sections"] | length > 0 and + is_empty | length > 0 + ) ) -%} {{ "\n" }} @@ -91,7 +97,12 @@ {%- elif item is string %} {#- Value is a string #} - {%- if " " in item or "\t" in item or "\n" in item or item == "" -%} + {%- if " " in item or + " " in item or + "\t" in item or + "\n" in item or + item in ["", "."] -%} + "{{ item | replace('"', '\\"') }}" {%- else -%} {{ item }} From e0d01afc52ab15e1ca41c05d51643fe3db3baa4b Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Thu, 12 Mar 2015 15:45:43 +0000 Subject: [PATCH 19/32] Adding quote_all_{nums,strings} options to the Apache macro --- macros/apache_encode_macro.j2 | 48 +++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/macros/apache_encode_macro.j2 b/macros/apache_encode_macro.j2 index 7ae13f9..7d347b3 100644 --- a/macros/apache_encode_macro.j2 +++ b/macros/apache_encode_macro.j2 @@ -8,6 +8,8 @@ convert_nums=false, indent=" ", level=0, + quote_all_nums=false, + quote_all_strings=false, type="sections") %} {%- if type == "sections" %} @@ -18,8 +20,11 @@ c["options"], convert_bools=convert_bools, convert_nums=convert_nums, - type="options", - level=level+1) }} + indent=indent, + level=level+1, + quote_all_nums=quote_all_nums, + quote_all_strings=quote_all_strings, + type="options") }} {%- endif %} {%- set is_empty = [] %} @@ -41,15 +46,21 @@ s["param"], convert_bools=convert_bools, convert_nums=convert_nums, - type="value", - level=level+1) + indent=indent, + level=level+1, + quote_all_nums=quote_all_nums, + quote_all_strings=quote_all_strings, + type="value") if "param" in s else "") ~ ">\n" ~ apache_encode( s, convert_bools=convert_bools, convert_nums=convert_nums, - type="sections", - level=level+1) ~ + indent=indent, + level=level+1, + quote_all_nums=quote_all_nums, + quote_all_strings=quote_all_strings, + type="sections") ~ indent * level ~ "\n" }} {%- if not loop.last -%} @@ -80,8 +91,11 @@ val, convert_bools=convert_bools, convert_nums=convert_nums, - type="value", - level=level+1) ~ "\n" }} + indent=indent, + level=level+1, + quote_all_nums=quote_all_nums, + quote_all_strings=quote_all_strings, + type="value") ~ "\n" }} {%- endfor %} {%- endfor %} @@ -93,15 +107,20 @@ {%- elif item is number %} {#- Value is a number -#} - {{ item }} + {%- if quote_all_nums -%} + "{{ item }}" + {%- else -%} + {{ item }} + {%- endif %} {%- elif item is string %} {#- Value is a string #} - {%- if " " in item or + {%- if quote_all_strings or " " in item or "\t" in item or "\n" in item or - item in ["", "."] -%} + "\r" in item or + item == "" -%} "{{ item | replace('"', '\\"') }}" {%- else -%} @@ -115,8 +134,11 @@ v, convert_bools=convert_bools, convert_nums=convert_nums, - type="value", - level=level+1) }} + indent=indent, + level=level+1, + quote_all_nums=quote_all_nums, + quote_all_strings=quote_all_strings, + type="value") }} {%- if not loop.last -%} {{ " " }} From ac0c539bb63c1eec911e839ef4c1c9d6d24cc61e Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Thu, 12 Mar 2015 22:06:00 +0000 Subject: [PATCH 20/32] Add missing parameter in recursion calls in the XML macro --- macros/xml_encode_macro.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/macros/xml_encode_macro.j2 b/macros/xml_encode_macro.j2 index fad5fcb..4e7bfa9 100644 --- a/macros/xml_encode_macro.j2 +++ b/macros/xml_encode_macro.j2 @@ -28,6 +28,7 @@ {#- Only if val is a list -#} {{ xml_encode( val, + first=first, indent=indent, level=level, prevkey=key) }} @@ -64,6 +65,7 @@ {%- for e in item -%} {{ indent * level }}<{{ prevkey }}>{{ xml_encode( e, + first=first, indent=indent, level=level+1, prevkey=prevkey) }} From 489d645cc4571f725e2ec4fdd9ec867fbba8a13a Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Fri, 13 Mar 2015 15:41:38 +0000 Subject: [PATCH 21/32] Adding proper numbers recognition --- macros/apache_encode_macro.j2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/macros/apache_encode_macro.j2 b/macros/apache_encode_macro.j2 index 7d347b3..2be0261 100644 --- a/macros/apache_encode_macro.j2 +++ b/macros/apache_encode_macro.j2 @@ -105,7 +105,11 @@ {#- Value is a boolean -#} {{ item | lower }} - {%- elif item is number %} + {%- elif item is number or + (convert_nums and ( + item | int | string == item or + item | float | string == item)) or + item in ["true", "True", "false", "False"] %} {#- Value is a number -#} {%- if quote_all_nums -%} "{{ item }}" From df9a8a4fbc048ab912026ffa3366e1812251dffa Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Wed, 18 Mar 2015 17:24:07 +0000 Subject: [PATCH 22/32] Multiply backslashes if there are any in the string --- macros/apache_encode_macro.j2 | 2 +- macros/yaml_encode_macro.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/apache_encode_macro.j2 b/macros/apache_encode_macro.j2 index 2be0261..6d80c07 100644 --- a/macros/apache_encode_macro.j2 +++ b/macros/apache_encode_macro.j2 @@ -126,7 +126,7 @@ "\r" in item or item == "" -%} - "{{ item | replace('"', '\\"') }}" + "{{ item | replace("\\", "\\\\") | replace('"', '\\"') }}" {%- else -%} {{ item }} {%- endif %} diff --git a/macros/yaml_encode_macro.j2 b/macros/yaml_encode_macro.j2 index 161f62b..1b4dbdb 100644 --- a/macros/yaml_encode_macro.j2 +++ b/macros/yaml_encode_macro.j2 @@ -51,7 +51,7 @@ {#- The item is a string -#} {{ " " + quote + - item | replace(quote, "\\" + quote) + + item | replace("\\", "\\\\") | replace(quote, "\\" + quote) + quote + "\n" }} {%- else %} From 4fc2c24ea9e85a3953abb2381bc5f7946d05b3a1 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Thu, 9 Apr 2015 15:31:35 +0100 Subject: [PATCH 23/32] Renaming the data variable --- README.md | 4 ++-- templates/test.apache.j2 | 2 +- templates/test.erlang.j2 | 2 +- templates/test.ini.j2 | 2 +- templates/test.ini_simple.j2 | 2 +- templates/test.json.j2 | 2 +- templates/test.toml.j2 | 2 +- templates/test.xml.j2 | 2 +- templates/test.yaml.j2 | 2 +- vars/apache_test.yaml | 2 +- vars/erlang_test.yaml | 2 +- vars/ini_test.yaml | 2 +- vars/json_test.yaml | 2 +- vars/toml_test.yaml | 2 +- vars/xml_test.yaml | 2 +- vars/yaml_test.yaml | 2 +- 16 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 01ac03c..2e96769 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ from your playbook (e.g. `site.yaml`) as follows: - name: Play to test the encoder macros hosts: all vars: - data: + ini_data: var1: val1 var2: val2 section1: @@ -70,7 +70,7 @@ Where the `templates/test.ini.j2` contains: {% from "macros/ini_encode_macro.j2" import ini_encode with context -%} -{{ ini_encode(data) }} +{{ ini_encode(ini_data) }} ``` And if you call the playbook: diff --git a/templates/test.apache.j2 b/templates/test.apache.j2 index 43fdb29..b8db502 100644 --- a/templates/test.apache.j2 +++ b/templates/test.apache.j2 @@ -6,4 +6,4 @@ {% from "macros/apache_encode_macro.j2" import apache_encode with context -%} -{{ apache_encode(data) }} +{{ apache_encode(apache_data) }} diff --git a/templates/test.erlang.j2 b/templates/test.erlang.j2 index 6088e8f..f1fcc51 100644 --- a/templates/test.erlang.j2 +++ b/templates/test.erlang.j2 @@ -6,4 +6,4 @@ {% from "macros/erlang_encode_macro.j2" import erlang_encode with context -%} -{{ erlang_encode(data) }} +{{ erlang_encode(erlang_data) }} diff --git a/templates/test.ini.j2 b/templates/test.ini.j2 index f5475b7..840c1c4 100644 --- a/templates/test.ini.j2 +++ b/templates/test.ini.j2 @@ -6,4 +6,4 @@ {% from "macros/ini_encode_macro.j2" import ini_encode with context -%} -{{ ini_encode(data) }} +{{ ini_encode(ini_data) }} diff --git a/templates/test.ini_simple.j2 b/templates/test.ini_simple.j2 index 648409c..551fd57 100644 --- a/templates/test.ini_simple.j2 +++ b/templates/test.ini_simple.j2 @@ -6,4 +6,4 @@ {% from "macros/ini_encode_macro.j2" import ini_encode with context -%} -{{ ini_encode(data, section_is_comment=true, delimiter=" ") }} +{{ ini_encode(ini_data, section_is_comment=true, delimiter=" ") }} diff --git a/templates/test.json.j2 b/templates/test.json.j2 index ddd802a..1962872 100644 --- a/templates/test.json.j2 +++ b/templates/test.json.j2 @@ -1,3 +1,3 @@ {% from "macros/json_encode_macro.j2" import json_encode with context -%} -{{ json_encode(data) }} +{{ json_encode(json_data) }} diff --git a/templates/test.toml.j2 b/templates/test.toml.j2 index 6a0d73e..909c892 100644 --- a/templates/test.toml.j2 +++ b/templates/test.toml.j2 @@ -6,4 +6,4 @@ {% from "macros/toml_encode_macro.j2" import toml_encode with context -%} -{{ toml_encode(data) }} +{{ toml_encode(toml_data) }} diff --git a/templates/test.xml.j2 b/templates/test.xml.j2 index 6d21b7d..d62ae88 100644 --- a/templates/test.xml.j2 +++ b/templates/test.xml.j2 @@ -10,4 +10,4 @@ {% from "macros/xml_encode_macro.j2" import xml_encode with context -%} -{{ xml_encode(data) }} +{{ xml_encode(xml_data) }} diff --git a/templates/test.yaml.j2 b/templates/test.yaml.j2 index 656539c..78e5efb 100644 --- a/templates/test.yaml.j2 +++ b/templates/test.yaml.j2 @@ -6,4 +6,4 @@ {% from "macros/yaml_encode_macro.j2" import yaml_encode with context -%} -{{ yaml_encode(data) }} +{{ yaml_encode(yaml_data) }} diff --git a/vars/apache_test.yaml b/vars/apache_test.yaml index 26095c7..59c3220 100644 --- a/vars/apache_test.yaml +++ b/vars/apache_test.yaml @@ -1,6 +1,6 @@ --- -data: +apache_data: content: - options: - ServerName: localhost diff --git a/vars/erlang_test.yaml b/vars/erlang_test.yaml index 268f54e..cec3ec8 100644 --- a/vars/erlang_test.yaml +++ b/vars/erlang_test.yaml @@ -1,6 +1,6 @@ --- -data: +erlang_data: - rabbit: - tcp_listeners: - '"127.0.0.1"': 5672 diff --git a/vars/ini_test.yaml b/vars/ini_test.yaml index 0314a0e..aa8c59d 100644 --- a/vars/ini_test.yaml +++ b/vars/ini_test.yaml @@ -1,6 +1,6 @@ --- -data: +ini_data: var1: val1 var2: val2 section1: diff --git a/vars/json_test.yaml b/vars/json_test.yaml index f40ea73..72fda5d 100644 --- a/vars/json_test.yaml +++ b/vars/json_test.yaml @@ -1,6 +1,6 @@ --- -data: +json_data: string: This is a string number: 123 boolean: 'true' diff --git a/vars/toml_test.yaml b/vars/toml_test.yaml index 623abab..041cebc 100644 --- a/vars/toml_test.yaml +++ b/vars/toml_test.yaml @@ -1,6 +1,6 @@ --- -data: +toml_data: title: TOML Example owner: name: Tom Preston-Werner diff --git a/vars/xml_test.yaml b/vars/xml_test.yaml index 3fce9dd..1707ee2 100644 --- a/vars/xml_test.yaml +++ b/vars/xml_test.yaml @@ -1,6 +1,6 @@ --- -data: +xml_data: root: elem1: asd elem2: diff --git a/vars/yaml_test.yaml b/vars/yaml_test.yaml index f40ea73..7d68a00 100644 --- a/vars/yaml_test.yaml +++ b/vars/yaml_test.yaml @@ -1,6 +1,6 @@ --- -data: +yaml_data: string: This is a string number: 123 boolean: 'true' From 936328dc79f7c0a0d1284bd4a751bf5c52bc58a8 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Thu, 9 Apr 2015 15:34:20 +0100 Subject: [PATCH 24/32] Adding epilog with examples --- yaml_converter.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/yaml_converter.py b/yaml_converter.py index 3c1e9e0..3d2b9e1 100755 --- a/yaml_converter.py +++ b/yaml_converter.py @@ -1,5 +1,6 @@ #!/usr/bin/env python2 +from argparse import RawTextHelpFormatter from jinja2 import Template from yaml import load import argparse @@ -8,8 +9,15 @@ def parse_arguments(): description = 'YAML to TOML converter' + epilog = ( + "Examples:\n" + " $ %(prog)s -f apache -v apache_data -y ./vars/apache_test.yaml\n" + " $ %(prog)s -f json -v json_data -y ./vars/json_test.yaml") - parser = argparse.ArgumentParser(description=description) + parser = argparse.ArgumentParser( + description=description, + epilog=epilog, + formatter_class=RawTextHelpFormatter) parser.add_argument( '--format', '-f', metavar='FORMAT', From dff8b18aad17e1b1ff79ce5fbbc188fd9659c56b Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Thu, 9 Apr 2015 15:36:06 +0100 Subject: [PATCH 25/32] Improving formating and fixing minor bugs --- macros/apache_encode_macro.j2 | 10 ++++++---- macros/erlang_encode_macro.j2 | 5 +++-- macros/ini_encode_macro.j2 | 14 +++++++++----- macros/json_encode_macro.j2 | 7 ++++--- macros/toml_encode_macro.j2 | 3 ++- macros/xml_encode_macro.j2 | 2 +- macros/yaml_encode_macro.j2 | 10 +++++----- 7 files changed, 30 insertions(+), 21 deletions(-) diff --git a/macros/apache_encode_macro.j2 b/macros/apache_encode_macro.j2 index 6d80c07..4f56d21 100644 --- a/macros/apache_encode_macro.j2 +++ b/macros/apache_encode_macro.j2 @@ -35,7 +35,8 @@ {#- Check for empty sub-sections #} {%- for i in s["content"] %} - {%- if ('options' in i and i['options'] | length > 0) or ('sections' in i and i['sections'] | length > 0) %} + {%- if ('options' in i and i['options'] | length > 0) or + ('sections' in i and i['sections'] | length > 0) %} {%- if is_empty.append(1) %}{% endif %} {%- endif %} {%- endfor %} @@ -85,7 +86,7 @@ {%- elif type == "options" %} {%- for o in item -%} - {%- for key, val in o.iteritems() -%} + {%- for key, val in o.iteritems() | sort -%} {{ indent * (level-1) ~ key ~ " " ~ apache_encode( val, @@ -100,7 +101,7 @@ {%- endfor %} {%- elif type == "value" %} - {%- if item == True or item == False or + {%- if item in [True, False] or convert_bools and (item in ["true", "True", "false", "False"]) %} {#- Value is a boolean -#} {{ item | lower }} @@ -109,7 +110,8 @@ (convert_nums and ( item | int | string == item or item | float | string == item)) or - item in ["true", "True", "false", "False"] %} + (convert_bools and item in ["true", "True", "false", "False"]) or + item in [True, False] %} {#- Value is a number -#} {%- if quote_all_nums -%} "{{ item }}" diff --git a/macros/erlang_encode_macro.j2 b/macros/erlang_encode_macro.j2 index 2b0639f..c15825a 100644 --- a/macros/erlang_encode_macro.j2 +++ b/macros/erlang_encode_macro.j2 @@ -18,7 +18,7 @@ {%- if first.append(1) %}{% endif %} {%- for key, val in item.iteritems() | sort -%} - {{ indent * level ~ "{" ~ key ~ ","}} + {{ indent * level ~ "{" ~ key ~ "," }} {%- if val is not mapping -%} {{ " " }} @@ -41,7 +41,8 @@ (convert_nums and ( item | int | string == item or item | float | string == item)) or - item in ["true", "True", "false", "False"] %} + (convert_bools and item in ["true", "True", "false", "False"]) or + item in [True, False] %} {#- Item is a value of a number, null or boolean -#} diff --git a/macros/ini_encode_macro.j2 b/macros/ini_encode_macro.j2 index 0f877e1..d525365 100644 --- a/macros/ini_encode_macro.j2 +++ b/macros/ini_encode_macro.j2 @@ -18,10 +18,12 @@ {%- endif -%} {%- if value is string or value is number -%} - {{ (property | upper if ucase_prop else property) ~ delimiter ~ quote ~ value ~ quote ~ "\n" }} + {{ (property | upper if ucase_prop else property) ~ + delimiter ~ quote ~ value ~ quote ~ "\n" }} {%- else %} {%- for item in value -%} - {{ (property | upper if ucase_prop else property) ~ delimiter ~ quote ~ item ~ quote ~ "\n" }} + {{ (property | upper if ucase_prop else property) ~ + delimiter ~ quote ~ item ~ quote ~ "\n" }} {%- endfor %} {%- endif %} {%- endif %} @@ -42,10 +44,12 @@ {%- endif %} {%- for property, value in options.iteritems() | sort -%} {%- if value is string or value is number -%} - {{ (property | upper if ucase_prop else property) ~ delimiter ~ quote ~ value ~ quote ~ "\n" }} + {{ (property | upper if ucase_prop else property) ~ + delimiter ~ quote ~ value ~ quote ~ "\n" }} {%- else %} - {%- for item in value -%} - {{ (property | upper if ucase_prop else property) ~ delimiter ~ quote ~ item ~ quote ~ "\n" }} + {%- for v in value -%} + {{ (property | upper if ucase_prop else property) ~ + delimiter ~ quote ~ v ~ quote ~ "\n" }} {%- endfor %} {%- endif %} {%- endfor %} diff --git a/macros/json_encode_macro.j2 b/macros/json_encode_macro.j2 index 2513466..b523432 100644 --- a/macros/json_encode_macro.j2 +++ b/macros/json_encode_macro.j2 @@ -19,7 +19,7 @@ {%- endif %} {%- for key, val in item.iteritems() | sort -%} - {{ indent * (level+1) ~ '"' ~ key ~ '": '}}{{ json_encode( + {{ indent * (level+1) ~ '"' ~ key ~ '": ' }}{{ json_encode( val, convert_bools=convert_bools, convert_nums=convert_nums, @@ -47,7 +47,8 @@ (convert_nums and ( item | int | string == item or item | float | string == item)) or - item in ["true", "True", "false", "False"] %} + (convert_bools and item in ["true", "True", "false", "False"]) or + item in [True, False] %} {#- Item is a value of a number, null or boolean -#} @@ -64,7 +65,7 @@ {{ "[" }} {%- if item | length > 0 -%} - {{ "\n" }} + {{ "\n" }} {%- endif %} {%- for val in item -%} diff --git a/macros/toml_encode_macro.j2 b/macros/toml_encode_macro.j2 index e4c801d..3205480 100644 --- a/macros/toml_encode_macro.j2 +++ b/macros/toml_encode_macro.j2 @@ -85,7 +85,8 @@ (convert_nums and ( item | int | string == item or item | float | string == item)) or - item in ["true", "True", "false", "False"] %} + (convert_bools and item in ["true", "True", "false", "False"]) or + item in [True, False] %} {#- The item is a number or boolean -#} {{ item | lower }}{{ "\n" }} diff --git a/macros/xml_encode_macro.j2 b/macros/xml_encode_macro.j2 index 4e7bfa9..514b631 100644 --- a/macros/xml_encode_macro.j2 +++ b/macros/xml_encode_macro.j2 @@ -51,7 +51,7 @@ {%- endif %} {%- endfor %} - {%- elif item is number or item is string %} + {%- elif item is number or item is string or item in [True, False] %} {#- The item is a number, string or boolean -#} {{ item }} diff --git a/macros/yaml_encode_macro.j2 b/macros/yaml_encode_macro.j2 index 1b4dbdb..ad1dbc3 100644 --- a/macros/yaml_encode_macro.j2 +++ b/macros/yaml_encode_macro.j2 @@ -13,7 +13,7 @@ {%- if item is mapping %} {#- The item is a dict #} - {%- for key, value in item.iteritems() | sort -%} + {%- for key, val in item.iteritems() | sort -%} {%- if skip_indent and loop.first -%} {{ " " }} {%- else -%} @@ -22,12 +22,12 @@ {{ key }}: - {%- if value is not string and value is not number -%} + {%- if val is not string and val is not number -%} {{ "\n" }} {%- endif -%} {{ yaml_encode( - value, + val, convert_bools=convert_bools, convert_nums=convert_nums, indent=indent, @@ -41,8 +41,8 @@ (convert_nums and ( item | int | string == item or item | float | string == item)) or - (convert_bools and ( - item in ["true", "True", "false", "False"])) %} + (convert_bools and item in ["true", "True", "false", "False"]) or + item in [True, False] %} {#- The item is a number or boolean -#} {{ " " ~ item | lower ~ "\n" }} From 3c20c32b6a504f87be8990f6066d35bb2a984771 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Mon, 13 Apr 2015 02:17:44 +0100 Subject: [PATCH 26/32] Improving formating and TOML macro --- README.md | 19 ++++++--- macros/toml_encode_macro.j2 | 80 ++++++++++++++++++++++++++----------- macros/xml_encode_macro.j2 | 2 +- templates/test.erlang.j2 | 10 ++--- 4 files changed, 77 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 2e96769..b722a73 100644 --- a/README.md +++ b/README.md @@ -289,6 +289,15 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > Defines the name of the previous key in the recursive macro calls. It's used > only internally in the macro. +- `prevtype=""` + + > Defines the type of the previous item in the recursive macro calls. It's used + > only internally in the macro. + +- `quote='"'` + + > Allows to set the *value* quoting. Use `quote="'"` or `quote='"'`. + ### `xml_encode()` - `item` @@ -421,11 +430,11 @@ import sys # Load the YAML data to Python data structure yaml_fh = open('vars/ini_test.yaml') -yaml_data = load(yaml_fh) +yaml_struct = load(yaml_fh) yaml_fh.close() -# Take only the content of the data variable -yaml_data = yaml_data['data'] +# Take only the content of the ini_data variable +yaml_struct = yaml_struct['ini_data'] # Read Jinja2 template as text template_fh = open('macros/ini_encode_macro.j2') @@ -435,7 +444,7 @@ template_fh.close() # Create Jinja2 template object t = Template(template_text) # Convert the YAML data to INI format -output = t.module.ini_encode(yaml_data).encode('utf8') +output = t.module.ini_encode(yaml_struct).encode('utf8') # Print the result sys.stdout.write(output) @@ -444,7 +453,7 @@ sys.stdout.write(output) Or you can use the `yaml_converter.py`: ``` -$ ./yaml_converter.py -f json -v data -y ./vars/json_test.yaml +$ ./yaml_converter.py -f ini -v ini_data -y ./vars/ini_test.yaml ``` diff --git a/macros/toml_encode_macro.j2 b/macros/toml_encode_macro.j2 index 3205480..c5cdfec 100644 --- a/macros/toml_encode_macro.j2 +++ b/macros/toml_encode_macro.j2 @@ -9,23 +9,26 @@ first=[], indent=" ", level=0, - prevkey="") %} + prevkey="", + prevtype="", + quote='"') %} {%- if item is mapping %} {#- The item is a dict #} - {#- First process all strings, numbers and lists #} - {%- for key, value in item.iteritems() | sort -%} - {%- if value is string or - value is number or + {#- First process all strings, numbers, booleans and lists #} + {%- for key, val in item.iteritems() | sort -%} + {%- if val is string or + val is number or + val in [True, False] or ( - value is not mapping and - value | length > 0 and - value[0] is not mapping) %} + val is not mapping and + val | length > 0 and + val[0] is not mapping) %} {#- The value is a string, a number, or a list -#} {{ indent * level }}{{ key }}{{ " = " }}{{ toml_encode( - value, + val, convert_bools=convert_bools, convert_nums=convert_nums, first=first, @@ -37,18 +40,19 @@ {%- endfor %} {#- Then process all data structures #} - {%- for key, value in item.iteritems() | sort %} - {%- if value is not string and - value is not number and + {%- for key, val in item.iteritems() | sort %} + {%- if val is not string and + val is not number and + val not in [True, False] and ( - value is mapping or - value[0] is mapping) %} + val is mapping or + val[0] is mapping) %} {%- set old_prevkey = prevkey %} {%- set old_level = level %} - {%- if value is mapping %} - {#- The value is a dict #} + {%- if val is mapping %} + {#- The val is a dict #} {%- if prevkey != "" and prevkey != key %} {%- set level = level + 1 %} {%- endif %} @@ -60,14 +64,14 @@ {%- endif -%} {{ indent * level }}[{{ prevkey }}]{{ "\n" }} - {%- elif value[0] is mapping %} - {#- The value is a table #} + {%- elif val[0] is mapping %} + {#- The val is a table #} {%- set prevkey = (key if prevkey == "" else prevkey + "." + key) %} - {%- set level = level +1 %} + {%- set level = level + 1 %} {%- endif -%} {{ toml_encode( - value, + val, convert_bools=convert_bools, convert_nums=convert_nums, first=first, @@ -89,11 +93,19 @@ item in [True, False] %} {#- The item is a number or boolean -#} - {{ item | lower }}{{ "\n" }} + {{ item | lower }} + + {%- if prevtype != "list" -%} + {{ "\n" }} + {%- endif -%} {%- elif item is string %} {#- The item is a string -#} - "{{ item | replace("\n", "\\n") | replace("\t", "\\t") }}"{{ "\n" }} + {{ quote ~ item | replace("\n", "\\n") | replace("\t", "\\t") ~ quote }} + + {%- if prevtype != "list" -%} + {{ "\n" }} + {%- endif -%} {%- else %} {#- The item is a list #} @@ -108,7 +120,29 @@ level=level) }} {%- endfor %} {%- else -%} - {{ item }}{{ "\n" }} + {{ "[" }} + + {%- for d in item -%} + {{ toml_encode( + d, + convert_bools=convert_bools, + convert_nums=convert_nums, + first=first, + indent=indent, + level=level, + prevtype="list") }} + + {%- if not loop.last -%} + {{ ", " }} + {%- endif %} + {%- endfor -%} + + {{ "]" }} + + {%- if prevtype != "list" -%} + {{ "\n" }} + {%- endif -%} + {%- endif %} {%- endif %} {%- endmacro %} diff --git a/macros/xml_encode_macro.j2 b/macros/xml_encode_macro.j2 index 514b631..33774ed 100644 --- a/macros/xml_encode_macro.j2 +++ b/macros/xml_encode_macro.j2 @@ -45,7 +45,7 @@ - {%- if loop.last-%} + {%- if loop.last -%} {{ "\n" }} {%- endif %} {%- endif %} diff --git a/templates/test.erlang.j2 b/templates/test.erlang.j2 index f1fcc51..ec06568 100644 --- a/templates/test.erlang.j2 +++ b/templates/test.erlang.j2 @@ -1,8 +1,8 @@ -%% -%% This file is managed by Ansible. -%% Do not edit this file manually. -%% Any changes will be automatically reverted. -%% +% +% This file is managed by Ansible. +% Do not edit this file manually. +% Any changes will be automatically reverted. +% {% from "macros/erlang_encode_macro.j2" import erlang_encode with context -%} From 4845257079ada3503d909c4f781fd29a8ace6e29 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Mon, 13 Apr 2015 02:47:04 +0100 Subject: [PATCH 27/32] Adding support for ERB and Puppet --- README.md | 171 +++++++++++++++++++++++++++++--- hiera.yaml | 12 +++ macros/apache_encode_macro.erb | 172 ++++++++++++++++++++++++++++++++ macros/erlang_encode_macro.erb | 82 +++++++++++++++ macros/ini_encode_macro.erb | 53 ++++++++++ macros/json_encode_macro.erb | 87 ++++++++++++++++ macros/toml_encode_macro.erb | 175 +++++++++++++++++++++++++++++++++ macros/xml_encode_macro.erb | 103 +++++++++++++++++++ macros/yaml_encode_macro.erb | 84 ++++++++++++++++ puppet_apply.sh | 13 +++ site.pp | 63 ++++++++++++ templates/test.apache.erb | 11 +++ templates/test.erlang.erb | 11 +++ templates/test.ini.erb | 10 ++ templates/test.ini_simple.erb | 12 +++ templates/test.json.erb | 5 + templates/test.toml.erb | 11 +++ templates/test.xml.erb | 15 +++ templates/test.yaml.erb | 11 +++ yaml_converter.py | 18 ++-- yaml_converter.rb | 112 +++++++++++++++++++++ 21 files changed, 1209 insertions(+), 22 deletions(-) create mode 100644 hiera.yaml create mode 100644 macros/apache_encode_macro.erb create mode 100644 macros/erlang_encode_macro.erb create mode 100644 macros/ini_encode_macro.erb create mode 100644 macros/json_encode_macro.erb create mode 100644 macros/toml_encode_macro.erb create mode 100644 macros/xml_encode_macro.erb create mode 100644 macros/yaml_encode_macro.erb create mode 100755 puppet_apply.sh create mode 100644 site.pp create mode 100644 templates/test.apache.erb create mode 100644 templates/test.erlang.erb create mode 100644 templates/test.ini.erb create mode 100644 templates/test.ini_simple.erb create mode 100644 templates/test.json.erb create mode 100644 templates/test.toml.erb create mode 100644 templates/test.xml.erb create mode 100644 templates/test.yaml.erb create mode 100755 yaml_converter.rb diff --git a/README.md b/README.md index b722a73..35aaad5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,22 @@ -Jinja2 Encoder Macros +Config Encoder Macros ===================== -Set of Jinja2 macros which help to encode Python data structure into a different -file format. +Set of Jinja2 and ERB macros which help to encode Python and Ruby data structure +into a different file format. Motivation ---------- The motivation to create these macros was to be able to create an universal -description of configuration files in [Ansible](http://ansible.com). Ansible is -using YAML format as the main data input which allows to describe almost any -structure of a configuration file. Ansible converts the YAML data into a Python -data structure using primitive data types (e.g. `dict`, `list`, `string`, -`number`, `boolean`, ...). That data structure is then used in Jinja2 template -files to produce the final configuration file. The encoder macros are used to -convert the Python data structure to another format. +description of configuration files in [Ansible](http://ansible.com). Ansible and +Puppet is using YAML format as the main data input. This format allows to +describe almost any structure of a configuration file. Ansible resp. Puppet +converts the YAML data into a Python resp. Ruby data structure using primitive +data types (e.g. `dict`, `list`, `string`, `number`, `boolean`, ...). That data +structure is then used in Jinja2 and ERB template files to produce the final +configuration file. The encoder macros are used to convert the Python or Ruby +data structure to another format. Supported formats @@ -33,8 +34,13 @@ Supported formats Ansible example --------------- -Clone this repo into the directory where your playbook is. Then use the macros -from your playbook (e.g. `site.yaml`) as follows: +Clone this repo into the `templates` directory in the playbook location: + +``` +$ git clone https://github.com/picotrading/config-encoder-macros ./templates/encoder +``` + +Then use the macros in your playbook (e.g. `site.yaml`) like this: ``` # Playbook example @@ -68,7 +74,7 @@ Where the `templates/test.ini.j2` contains: # Any changes will be automatically reverted. # -{% from "macros/ini_encode_macro.j2" import ini_encode with context -%} +{% from "templates/encoder/macros/ini_encode_macro.j2" import ini_encode with context -%} {{ ini_encode(ini_data) }} ``` @@ -105,6 +111,89 @@ fff=false Look into the `site.yaml` and `templates/*.j2` for more examples. +Puppet example +-------------- + +Clone this repo to some directory (e.g. `/etc/puppet/encoder`): + +``` +$ git clone https://github.com/picotrading/config-encoder-macros /etc/puppet/encoder +``` + +Then use the macros from your Puppet manifest like this: + +``` +# Load the configuration from YAML file +$ini_data = hiera('ini_data') + +file { '/tmp/test.ini' : + ensure => present, + content => template('test.ini.erb'), +} +``` + +Where the Hiera YAML file contains: + +``` +--- + +ini_data: + var1: val1 + var2: val2 + section1: + aaa: + - asdf + - zxcv + bbb: 123 + ccc: 'true' + section2: + ddd: asdfasd + eee: 1234 + fff: 'false' +``` + +And the `templates/test.ini.erb` contains: + +``` +# +# This file is managed by Puppet. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +<%- + item = @ini_data || (ini_data.kind_of?(String) ? eval(ini_data) : ini_data) +-%> +<%= ERB.new(IO.read('/etc/puppet/macros/ini_encode_macro.erb'), nil, '-', '_erbout1').result(OpenStruct.new().send(:binding)) -%> +``` + +And if you run Puppet you should get the following in the `/tmp/test.ini`: + +``` +# +# This file is managed by Puppet. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +var1=val1 +var2=val2 + +[section1] +aaa=asdf +bbb=123 +ccc=true + +[section2] +ddd=asdfasd +eee=1234 +fff=false +``` + +Look into the `site.pp`, `templates/*.erb` and `puppet_apply.sh` for more +examples. + + Macros parameters ----------------- @@ -149,6 +238,10 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > from the beginning of the line. Setting the value to higher than `0` > indents the content by `indent * level`. +- `macro_path='macros/apache_encode_macro.erb'` + + > Relative or absolute path to the macro. It's used only in ERB macros. + ### `erlang_encode()` @@ -186,6 +279,10 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > from the beginning of the line. Setting the value to higher than `0` > indents the content by `indent * level`. +- `macro_path='macros/erlang_encode_macro.erb'` + + > Relative or absolute path to the macro. It's used only in ERB macros. + ### `ini_encode()` - `item` @@ -202,6 +299,10 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > Indicates whether the item being processed is actually a first item. It's > used only internally in the macro. +- `macro_path='macros/ini_encode_macro.erb'` + + > Relative or absolute path to the macro. It's used only in ERB macros. + - `quote=""` > Allows to set the *value* quoting. Use `quote="'"` or `quote='"'`. @@ -245,6 +346,10 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > from the beginning of the line. Setting the value to higher than `0` > indents the content by `indent * level`. +- `macro_path='macros/json_encode_macro.erb'` + + > Relative or absolute path to the macro. It's used only in ERB macros. + ### `toml_encode()` @@ -284,6 +389,10 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > from the beginning of the line. Setting the value to higher than `0` > indents the content by `indent * level`. +- `macro_path='macros/toml_encode_macro.erb'` + + > Relative or absolute path to the macro. It's used only in ERB macros. + - `prevkey=""` > Defines the name of the previous key in the recursive macro calls. It's used @@ -319,6 +428,10 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > from the beginning of the line. Setting the value to higher than `0` > indents the content by `indent * level`. +- `macro_path='macros/xml_encode_macro.erb'` + + > Relative or absolute path to the macro. It's used only in ERB macros. + - `prevkey=none` > Defines the name of the previous key in the recursive macro calls. It's used @@ -355,6 +468,10 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > from the beginning of the line. Setting the value to higher than `0` > indents the content by `indent * level`. +- `macro_path='macros/yaml_encode_macro.erb'` + + > Relative or absolute path to the macro. It's used only in ERB macros. + - `quote='"'` > Allows to set the *value* quoting. Use `quote="'"` or `quote='"'`. @@ -457,6 +574,34 @@ $ ./yaml_converter.py -f ini -v ini_data -y ./vars/ini_test.yaml ``` +Ruby example +------------ + +``` +require 'erb' +require 'ostruct' +require 'yaml' + +# Helper variables +yaml_file = './vars/ini_test.yaml' +var = 'ini_data' + +# Define variables used in the macro +item = YAML.load_file(yaml_file)[var] +macro_path = './macros/ini_encode_macro.erb' + +# Convert the YAML data to the final format +binding = OpenStruct.new.send(:binding) +print ERB.new(IO.read(macro_path), nil, '-', '_erbout0').result(binding) +``` + +Or you can use the `yaml_converter.rb`: + +``` +$ ./yaml_converter.rb -f ini -v ini_data -y ./vars/ini_test.yaml +``` + + License ------- diff --git a/hiera.yaml b/hiera.yaml new file mode 100644 index 0000000..6c4478b --- /dev/null +++ b/hiera.yaml @@ -0,0 +1,12 @@ +:backends: + - yaml +:hierarchy: + - apache_test + - erlang_test + - ini_test + - json_test + - toml_test + - xml_test + - yaml_test +:yaml: + :datadir: ./vars diff --git a/macros/apache_encode_macro.erb b/macros/apache_encode_macro.erb new file mode 100644 index 0000000..1ad36fc --- /dev/null +++ b/macros/apache_encode_macro.erb @@ -0,0 +1,172 @@ +<%-# + # ERB macro which converts Ruby data structure to Apache format + #-%> +<%- + _item ||= Array.new + _type ||= Array.new + _is_empty ||= Array.new + convert_bools ||= false + convert_nums ||= false + indent ||= ' ' + level ||= 0 + macro_path ||= 'macros/apache_encode_macro.erb' + quote_all_nums ||= false + quote_all_strings ||= false + type ||= 'sections' +-%> +<%- if type == 'sections' -%> + <%- item['content'].each do |c| -%> + <%-# First check if this section has options -%> + <%- if c.include?('options') -%> + <%- + _item.push(item) + _type.push(type) + item = c['options'] + type = 'options' + level = level+1 + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + item = _item.pop() + type = _type.pop() + level = level-1 + -%> + <%- end -%> + <%- is_empty = [] -%> + <%-# Check if this section has some sub-sections -%> + <%- if c.include?('sections') -%> + <%- c['sections'].each do |s| -%> + <%-# Check for empty sub-sections -%> + <%- s['content'].each do |i| -%> + <%- if (i.include?('options') and i['options'].length > 0) or + (i.include?('sections') and i['sections'].length > 0) -%> + <%- is_empty.push(1) -%> + <%- end -%> + <%- end -%> + <%- if is_empty.length > 0 -%> + <%--%><%= indent * level + '<' + s['name'] -%> + <%- if s.include?('param') -%> + <%--%><%= ' ' -%> + <%- + _item.push(item) + _type.push(type) + item = s['param'] + type = 'value' + level = level+1 + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + item = _item.pop() + type = _type.pop() + level = level-1 + -%> + <%- end -%> + <%--%><%= ">\n" -%> + <%- + _item.push(item) + _type.push(type) + item = s + type = 'sections' + level = level+1 + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + item = _item.pop() + type = _type.pop() + level = level-1 + -%> + <%--%><%= indent * level + '\n" -%> + <%- if c != item['content'].last -%> + <%--%><%= "\n" -%> + <%- end -%> + <%- end -%> + <%- end -%> + <%- end -%> + <%- if c != item['content'].last and + ( + c.include?('options') and c['options'].length > 0 or + ( + c.include?('sections') and c['sections'].length > 0 and + is_empty.length > 0 + ) + ) + -%> + <%--%><%= "\n" -%> + <%- end -%> + <%- end -%> +<%- elsif type == 'options' -%> + <%- item.each do |o| -%> + <%- o.sort.each do |key, val| -%> + <%--%><%= indent * (level-1) + key + ' ' -%> + <%- + _item.push(item) + _type.push(type) + item = val + type = 'value' + level = level+1 + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + item = _item.pop() + type = _type.pop() + level = level-1 + -%> + <%--%><%= "\n" -%> + <%- end -%> + <%- end -%> +<%- elsif type == 'value' -%> + <%- if item.kind_of?(TrueClass) or item.kind_of?(FalseClass) or + convert_bools and ['true', 'True', 'false', 'False'].include?(item) -%> + <%-# Value is a boolean -%> + <%--%><%= item.to_s.downcase -%> + <%- elsif item.kind_of?(Numeric) or + (convert_nums and ( + item.to_i.to_s == item or + item.to_f.to_s == item)) or + (convert_bools and ['true', 'True', 'false', 'False'].include?(item)) or + item.kind_of?(TrueClass) or item.kind_of?(FalseClass) -%> + <%-# Value is a number -%> + <%- if quote_all_nums -%> + <%--%><%= '"' + item.to_s + '"' -%> + <%- else -%> + <%--%><%= item -%> + <%- end -%> + <%- elsif item.kind_of?(String) -%> + <%-# Value is a string -%> + <%- if quote_all_strings or + item.include?(' ') or + item.include?("\t") or + item.include?("\n") or + item.include?("\r") or + item == '' -%> + <%--%><%= '"' + item.to_s.gsub('\\', '\\\\').gsub('"', '\\"') + '"' -%> + <%- else -%> + <%--%><%= item -%> + <%- end -%> + <%- elsif ! item.kind_of?(Hash) -%> + <%-# Value is an array -%> + <%- item.each do |v| -%> + <%- + _item.push(item) + _type.push(type) + item = v + type = 'value' + level = level+1 + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + item = _item.pop() + type = _type.pop() + level = level-1 + -%> + <%- if v != item.last -%> + <%--%><%= ' ' -%> + <%- end -%> + <%- end -%> + <%- end -%> +<%- end -%> diff --git a/macros/erlang_encode_macro.erb b/macros/erlang_encode_macro.erb new file mode 100644 index 0000000..f911ce3 --- /dev/null +++ b/macros/erlang_encode_macro.erb @@ -0,0 +1,82 @@ +<%-# + # ERB macro which converts Ruby data structure to Erlang config format + #-%> +<%- + _item ||= Array.new + convert_bools ||= false + convert_nums ||= false + first ||= Array.new + indent ||= ' ' + level ||= 0 + macro_path ||= 'macros/erlang_encode_macro.erb' +-%> +<%- if item.kind_of?(Hash) -%> + <%-# The item is a hash #-%> + <%--%><%= "\n" -%> + <%- first.push(1) -%> + <%- item.sort.each do |key, val| -%> + <%--%><%= indent * level + '{' + key.to_s + ',' -%> + <%- if ! val.kind_of?(Hash) -%> + <%--%><%= ' ' -%> + <%- end -%> + <%- + _item.push(item) + + item = val + level = level+1 + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + level = level-1 + item = _item.pop() + -%> + <%--%><%= '}' -%> + <%- if key != item.keys.sort.last -%> + <%--%><%= ",\n" -%> + <%- end -%> + <%- end -%> +<%- elsif item.nil? or item.kind_of?(Numeric) or + (convert_nums && ( + item.to_i.to_s == item or + item.to_f.to_s == item)) or + (convert_bools && ['true', 'True', 'false', 'False'].include?(item)) or + item.kind_of?(TrueClass) or item.kind_of?(FalseClass) -%> + <%-# Item is a value of a number, null or boolean #-%> + <%--%><%= item.to_s.downcase -%> +<%- elsif item.kind_of?(String) -%> + <%-# The item is a string #-%> + <%--%><%= '"' + item.to_s + '"' -%> +<%- else -%> + <%-# The item is an array #-%> + <%--%><%= '[' -%> + <%- item.each do |val| -%> + <%- if val.kind_of?(String) or val.kind_of?(Numeric) -%> + <%--%><%= "\n" + indent * (level) -%> + <%- end -%> + <%- + _item.push(item) + item = val + level = level+1 + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + level = level-1 + item = _item.pop() + -%> + <%- if val == item.last -%> + <%--%><%= "\n" -%> + <%- else -%> + <%--%><%= ',' -%> + <%- end -%> + <%- end -%> + <%- if item.length > 0 -%> + <%--%><%= indent * (level-1 > 0 ? level-1 : 0) + ']' -%> + <%- else -%> + <%--%><%= ']' -%> + <%- end -%> + <%- if level == 0 -%> + <%--%><%= ".\n" -%> + <%- end -%> +<%- end -%> diff --git a/macros/ini_encode_macro.erb b/macros/ini_encode_macro.erb new file mode 100644 index 0000000..d024652 --- /dev/null +++ b/macros/ini_encode_macro.erb @@ -0,0 +1,53 @@ +<%# + # ERB macro which converts Ruby data structure to INI format + #-%> +<%- + item ||= {} + delimiter ||= '=' + first ||= Array.new + quote ||= '' + section_is_comment ||= false + ucase_prop ||= false -%> +<%-# First process vars without a section -%> +<%- item.sort.each do |property, value| -%> + <%- if ! value.kind_of?(Hash) -%> + <%- if first.length == 0 -%> + <%- first.push(1) -%> + <%- end -%> + <%- if value.kind_of?(String) or value.kind_of?(Numeric) -%> + <%--%><%= (ucase_prop ? property.to_s.upcase : property) + delimiter + + quote + value.to_s + quote + "\n" -%> + <%- else -%> + <%- value.each do |i| -%> + <%--%><%= (ucase_prop ? property.to_s.upcase : property) + delimiter + + quote + i.to_s + quote + "\n" -%> + <%- end -%> + <%- end -%> + <%- end -%> +<%- end -%> +<%#- Then process vars with a section -%> +<%- item.sort.each do |section, options| -%> + <%- if options.kind_of?(Hash) -%> + <%- if first.length == 0 -%> + <%- first.push(1) -%> + <%- else -%> + <%--%><%= "\n" -%> + <%- end -%> + <%- if section_is_comment -%> + <%--%><%= '# ' + section.to_s + "\n" -%> + <%- else -%> + <%--%><%= '[' + section.to_s + "]\n" -%> + <%- end -%> + <%- options.sort.each do |property, value| -%> + <%- if value.kind_of?(String) or value.kind_of?(Numeric) -%> + <%--%><%= (ucase_prop ? property.to_s.upcase : property) + delimiter + + quote + value.to_s + quote + "\n" -%> + <%- else -%> + <%- value.each do |v| -%> + <%--%><%= (ucase_prop ? property.to_s.upcase : property) + + delimiter + quote + v.to_s + quote + "\n" -%> + <%- end -%> + <%- end -%> + <%- end -%> + <%- end -%> +<%- end -%> diff --git a/macros/json_encode_macro.erb b/macros/json_encode_macro.erb new file mode 100644 index 0000000..db4e088 --- /dev/null +++ b/macros/json_encode_macro.erb @@ -0,0 +1,87 @@ +<%-# + # ERB macro which converts Ruby data structure to JSON format + #-%> +<%- + _item ||= Array.new + convert_bools ||= false + convert_nums ||= false + indent ||= ' ' + level ||= 0 + macro_path ||= 'macros/json_encode_macro.erb' +-%> +<%- if item.kind_of?(Hash) -%> + <%-# The item is a hash -%> + <%--%><%= '{' -%> + <%- if item.length > 0 -%> + <%--%><%= "\n" -%> + <%- end -%> + <%- item.sort.each do |key, val| -%> + <%--%><%= indent * (level+1) + '"' + key.to_s + '": ' -%> + <%- + _item.push(item) + + item = val + level = level+1 + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + level = level-1 + item = _item.pop() + -%> + <%- if key == item.keys.sort.last -%> + <%--%><%= "\n" -%> + <%- else -%> + <%--%><%= ",\n" -%> + <%- end -%> + <%- end -%> + <%- if item.length > 0 -%> + <%--%><%= indent * level + '}' -%> + <%- else -%> + <%--%><%= '}' -%> + <%- end -%> + <%- if level == 0 -%> + <%--%><%= "\n" -%> + <%- end -%> +<%- elsif item.kind_of?(Numeric) or + (convert_nums and ( + item.to_i.to_s == item or + item.to_f.to_s == item)) or + (convert_bools and ['true', 'True', 'false', 'False'].include?(item)) or + item.kind_of?(TrueClass) or item.kind_of?(FalseClass) -%> + <%-# Item is a value of a number, null or boolean -%> + <%--%><%= item.to_s -%> +<%- elsif item.kind_of?(String) -%> + <%-# The item is a string -%> + <%--%><%= '"' + item.to_s + '"' -%> +<%- else -%> + <%-# The item is an array -%> + <%--%><%= '[' -%> + <%- if item.length > 0 -%> + <%--%><%= "\n" -%> + <%- end -%> + <%- item.each do |val| -%> + <%--%><%= indent * (level+1) -%> + <%- + _item.push(item) + item = val + level = level+1 + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + level = level-1 + item = _item.pop() + -%> + <%- if val == item.last -%> + <%--%><%= "\n" -%> + <%- else -%> + <%--%><%= ",\n" -%> + <%- end -%> + <%- end -%> + <%- if item.length > 0 -%> + <%--%><%= indent * level + ']' -%> + <%- else -%> + <%--%><%= ']' -%> + <%- end -%> +<%- end -%> diff --git a/macros/toml_encode_macro.erb b/macros/toml_encode_macro.erb new file mode 100644 index 0000000..cfcb6fd --- /dev/null +++ b/macros/toml_encode_macro.erb @@ -0,0 +1,175 @@ +<%-# + # ERB macro which converts Ruby data structure to TOML format + #-%> +<%- + _item ||= Array.new + _prevkey ||= Array.new + _prevtype ||= Array.new + _old_prevkey ||= Array.new + _old_level ||= Array.new + convert_bools ||= false + convert_nums ||= false + first ||= Array.new + indent ||= ' ' + level ||= 0 + macro_path ||= 'macros/toml_encode_macro.erb' + quote ||= '"' + prevkey ||= '' + prevtype ||= '' +-%> +<%- if item.kind_of?(Hash) -%> + <%-# The item is a hash -%> + <%-# First process all strings, numbers, booleans and arrays -%> + <%- item.sort.each do |key, val| -%> + <%- if val.kind_of?(String) or + val.kind_of?(Numeric) or + val.kind_of?(TrueClass) or val.kind_of?(FalseClass) or + ( + ! val.kind_of?(Hash) and + val.length > 0 and + ! val[0].kind_of?(Hash)) -%> + <%-# The value is a string, a number, or an array -%> + <%--%><%= indent * level + key.to_s + ' = ' -%> + <%- + _item.push(item) + _prevkey.push(prevkey) + _prevtype.push(prevtype) + item = val + prevtype = '' + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + prevkey = _prevkey.pop() + prevtype = _prevtype.pop() + item = _item.pop() + -%> + <%- first.push(1) -%> + <%- end -%> + <%- end -%> + <%-# Then process all data structures -%> + <%- item.sort.each do |key, val| -%> + <%- if ! val.kind_of?(String) and + ! val.kind_of?(Numeric) and + ! val.kind_of?(TrueClass) and ! val.kind_of?(FalseClass) and + (val.kind_of?(Hash) or val[0].kind_of?(Hash)) -%> + <%- + old_prevkey = prevkey + old_level = level + -%> + <%- if val.kind_of?(Hash) -%> + <%-# The value is a hash -%> + <%- if prevkey != '' and prevkey != key -%> + <%- level = level + 1 -%> + <%- end -%> + <%- if prevkey == '' -%> + <%- prevkey = key -%> + <%- else -%> + <%- prevkey = prevkey + '.' + key -%> + <%- end -%> + <%- if first.length > 0 -%> + <%--%><%= "\n" -%> + <%- end -%> + <%--%><%= indent * level + '[' + prevkey + ']' + "\n" -%> + <%- elsif val[0].kind_of?(Hash) -%> + <%-# The value is a table -%> + <%- if prevkey == '' -%> + <%- prevkey = key -%> + <%- else -%> + <%- prevkey = prevkey + '.' + key.to_s -%> + <%- end -%> + <%- level = level + 1 -%> + <%- end -%> + <%- + _item.push(item) + _prevkey.push(prevkey) + _prevtype.push(prevtype) + _old_prevkey.push(old_prevkey) + _old_level.push(old_level) + item = val + prevtype = '' + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + prevkey = _prevkey.pop() + prevtype = _prevtype.pop() + old_prevkey = _old_prevkey.pop() + old_level = _old_level.pop() + item = _item.pop() + -%> + <%- if val.kind_of?(Hash) and prevkey != '' and prevkey != key or + val[0].kind_of?(Hash) -%> + <%- level = level - 1 -%> + <%- end -%> + <%- + prevkey = old_prevkey + level = old_level + -%> + <%- first.push(1) -%> + <%- end -%> + <%- end -%> +<%- elsif item.kind_of?(Numeric) or + (convert_nums and ( + item.to_i.to_s == item or + item.to_f.to_s == item)) or + (convert_bools and ['true', 'True', 'false', 'False'].include?(item)) or + item.kind_of?(TrueClass) or item.kind_of?(FalseClass) -%> + <%-# The item is a number or boolean -%> + <%--%><%= item.to_s.downcase -%> + <%- if prevtype != 'array' -%> + <%--%><%= "\n" -%> + <%- end -%> +<%- elsif item.kind_of?(String) -%> + <%-# The item is a string -%> + <%--%><%= quote + item.to_s.gsub("\n", "\\n").gsub("\t", "\\t") + quote -%> + <%- if prevtype != 'array' -%> + <%--%><%= "\n" -%> + <%- end -%> +<%- else -%> + <%-# The item is an array -%> + <%- if item[0].kind_of?(Hash) -%> + <%- item.each do |d| -%> + <%--%><%= "\n" + indent * level + '[[' + prevkey.to_s + ']]' + "\n" -%> + <%- + _item.push(item) + _prevkey.push(prevkey) + _prevtype.push(prevtype) + item = d + prevtype = 'array' + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + prevkey = _prevkey.pop() + prevtype = _prevtype.pop() + item = _item.pop() + -%> + <%- end -%> + <%- else -%> + <%--%><%= '[' -%> + <%- item.each do |d| -%> + <%- + _item.push(item) + _prevkey.push(prevkey) + _prevtype.push(prevtype) + item = d + prevtype = 'array' + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + prevkey = _prevkey.pop() + prevtype = _prevtype.pop() + item = _item.pop() + -%> + <%- if d != item.last -%> + <%--%><%= ', ' -%> + <%- end -%> + <%- end -%> + <%--%><%= ']' -%> + <%- if prevtype != 'array' -%> + <%--%><%= "\n" -%> + <%- end -%> + <%- end -%> +<%- end -%> diff --git a/macros/xml_encode_macro.erb b/macros/xml_encode_macro.erb new file mode 100644 index 0000000..69b82be --- /dev/null +++ b/macros/xml_encode_macro.erb @@ -0,0 +1,103 @@ +<%-# + # ERB macro which converts Ruby data structure to XML format + #-%> +<%- + _item ||= Array.new + _prev_list ||= Array.new + _prevkey ||= Array.new + first ||= Array.new + indent ||= ' ' + level ||= 0 + macro_path ||= 'macros/xml_encode_macro.erb' + prevkey ||= nil +-%> +<%- if item.kind_of?(Hash) -%> + <%-# The item is a hash -%> + <%- prev_list = [0] -%> + <%- item.sort.each do |key, val| -%> + <%- if first.length > 0 and prev_list[0] == 0 -%> + <%--%><%= "\n" -%> + <%- end -%> + <%- first.push(0) -%> + <%- + prev_list.insert(0, 0) + prev_list.delete_at(1) + -%> + <%- if val == nil -%> + <%--%><%= indent * level + '<' + key.to_s + '/>' -%> + <%- elsif val.kind_of?(Array) -%> + <%-# Only if val is an array -%> + <%- + _item.push(item) + _prevkey.push(prevkey) + _prev_list.push(prev_list) + prev_list = [0] + prevkey = key + item = val + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + prevkey = _prevkey.pop() + prev_list = _prev_list.pop() + item = _item.pop() + -%> + <%- + prev_list.insert(0, 1) + prev_list.delete_at(1) + -%> + <%- else -%> + <%--%><%= indent * level + '<' + key.to_s + '>' -%> + <%- + _item.push(item) + _prevkey.push(prevkey) + _prev_list.push(prev_list) + prev_list = [0] + prevkey = key + item = val + level = level+1 + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + prevkey = _prevkey.pop() + prev_list = _prev_list.pop() + item = _item.pop() + level = level-1 + -%> + <%- if val.kind_of?(Hash) or val.kind_of?(Array) -%> + <%--%><%= indent * level -%> + <%- end -%> + <%--%><%= '' -%> + <%- if key == item.keys.sort.last -%> + <%--%><%= "\n" -%> + <%- end -%> + <%- end -%> + <%- end -%> +<%- elsif item.kind_of?(Numeric) or item.kind_of?(String) or + item.kind_of?(TrueClass) or item.kind_of?(FalseClass) -%> + <%-# The item is a number, string or boolean -%> + <%--%><%= item -%> +<%- elsif item == nil -%> + <%-# Item is a value of an empty element -%> +<%- else -%> + <%-# The item is an array -%> + <%- item.each do |e| -%> + <%--%><%= indent * level + '<' + prevkey + '>' -%> + <%- + _item.push(item) + item = e + level = level+1 + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + item = _item.pop() + level = level-1 + -%> + <%- if ! e.kind_of?(String) and ! e.kind_of?(Numeric) and e != nil -%> + <%--%><%= indent * level -%> + <%- end -%> + <%--%><%= '' + "\n" -%> + <%- end -%> +<%- end -%> diff --git a/macros/yaml_encode_macro.erb b/macros/yaml_encode_macro.erb new file mode 100644 index 0000000..7cd49f8 --- /dev/null +++ b/macros/yaml_encode_macro.erb @@ -0,0 +1,84 @@ +<%-# + # ERB macro which converts Ruby data structure to YAML format + #-%> +<%- + _item ||= Array.new + _skip_indent ||= Array.new + convert_bools ||= false + convert_nums ||= false + indent ||= ' ' + level ||= 0 + macro_path ||= 'macros/yaml_encode_macro.erb' + quote ||= '"' + skip_indent ||= false +-%> +<%- if item.kind_of?(Hash) -%> + <%-# The item is a hash -%> + <%- item.sort.each do |key, val| -%> + <%- if skip_indent and key == item.keys.sort.first -%> + <%--%><%= ' ' -%> + <%- else -%> + <%--%><%= indent * level -%> + <%- end -%> + <%--%><%= key.to_s + ':' -%> + <%- if ! val.kind_of?(String) and + ! val.kind_of?(Numeric) and + ! val.kind_of?(TrueClass) and + ! val.kind_of?(FalseClass)-%> + <%--%><%= "\n" -%> + <%- end -%> + <%- + _item.push(item) + item = val + level = level+1 + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + level = level-1 + item = _item.pop() + -%> + <%- end -%> + <%- if item.length == 0 -%> + <%--%><%= indent * level + "{ }\n" -%> + <%- end -%> +<%- elsif item.kind_of?(Numeric) or + (convert_nums and ( + item.to_i.to_s == item or + item.to_f.to_s == item)) or + (convert_bools and ['true', 'True', 'false', 'False'].include?(item)) or + item.kind_of?(TrueClass) or item.kind_of?(FalseClass) -%> + <%-# The item is a number or boolean -%> + <%--%><%= ' ' + item.to_s.downcase + "\n" -%> +<%- elsif item.kind_of?(String) -%> + <%-# The item is a string -%> + <%--%><%= ' ' + + quote + + item.gsub('\\', '\\\\').gsub(quote, '\\' + quote) + + quote + "\n" -%> +<%- else -%> + <%-# The item is an array -%> + <%- item.each do |n| -%> + <%--%><%= indent * level + '-' -%> + <%- if ! n.kind_of?(String) and + ! n.kind_of?(Numeric) and + ! n.kind_of?(Hash) -%> + <%--%><%= "\n" -%> + <%- end -%> + <%- + _item.push(item) + _skip_indent.push(skip_indent) + + item = n + level = level+1 + skip_indent = true + -%> + <%--%><%= ERB.new(IO.read(macro_path), nil, '-', '_erbout_' + + rand(36**20).to_s(36)).result(OpenStruct.new().send(:binding)) -%> + <%- + skip_indent = _skip_indent.pop() + level = level-1 + item = _item.pop() + -%> + <%- end -%> +<%- end -%> diff --git a/puppet_apply.sh b/puppet_apply.sh new file mode 100755 index 0000000..619b518 --- /dev/null +++ b/puppet_apply.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Apply Puppet manifests +puppet apply \ + --test \ + --confdir=. \ + --ssldir=/tmp/puppet_ssl \ + --templatedir=./templates \ + $@ \ + ./site.pp + +# Or you can use ERB directly: +# $ erb -r ostruct -T '-' 'ini_data={"aaa" => "bbb", "ccc" => {"ddd" => "eee"}}' ./templates/test.ini.erb diff --git a/site.pp b/site.pp new file mode 100644 index 0000000..f3e3970 --- /dev/null +++ b/site.pp @@ -0,0 +1,63 @@ +node default { + ### Apache + $apache_data = hiera('apache_data') + + file { '/tmp/test.apache' : + ensure => present, + content => template('test.apache.erb'), + } + + ### Erlang + $erlang_data = hiera('erlang_data') + + file { '/tmp/test.erlang' : + ensure => present, + content => template('test.erlang.erb'), + } + + ### INI + $ini_data = hiera('ini_data') + + file { '/tmp/test.ini' : + ensure => present, + content => template('test.ini.erb'), + } + + file { '/tmp/test.simple' : + ensure => present, + content => template('test.ini_simple.erb'), + } + + + ### JSON + $json_data = hiera('json_data') + + file { '/tmp/test.json' : + ensure => present, + content => template('test.json.erb'), + } + + ### TOML + $toml_data = hiera('toml_data') + + file { '/tmp/test.toml' : + ensure => present, + content => template('test.toml.erb'), + } + + ### XML + $xml_data = hiera('xml_data') + + file { '/tmp/test.xml' : + ensure => present, + content => template('test.xml.erb'), + } + + ### YAML + $yaml_data = hiera('yaml_data') + + file { '/tmp/test.yaml' : + ensure => present, + content => template('test.yaml.erb'), + } +} diff --git a/templates/test.apache.erb b/templates/test.apache.erb new file mode 100644 index 0000000..6290ec9 --- /dev/null +++ b/templates/test.apache.erb @@ -0,0 +1,11 @@ +# +# This file is managed by Puppet. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +<%- + item = @apache_data || (apache_data.kind_of?(String) ? eval(apache_data) : apache_data) + macro_path = 'macros/apache_encode_macro.erb' +-%> +<%= ERB.new(IO.read(macro_path), nil, '-', '_erbout1').result(OpenStruct.new().send(:binding)) -%> diff --git a/templates/test.erlang.erb b/templates/test.erlang.erb new file mode 100644 index 0000000..575af7f --- /dev/null +++ b/templates/test.erlang.erb @@ -0,0 +1,11 @@ +% +% This file is managed by Puppet. +% Do not edit this file manually. +% Any changes will be automatically reverted. +% + +<%- + item = @erlang_data || (erlang_data.kind_of?(String) ? eval(erlang_data) : erlang_data) + macro_path = 'macros/erlang_encode_macro.erb' +-%> +<%= ERB.new(IO.read(macro_path), nil, '-', '_erbout1').result(OpenStruct.new().send(:binding)) -%> diff --git a/templates/test.ini.erb b/templates/test.ini.erb new file mode 100644 index 0000000..bc60dab --- /dev/null +++ b/templates/test.ini.erb @@ -0,0 +1,10 @@ +# +# This file is managed by Puppet. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +<%- + item = @ini_data || (ini_data.kind_of?(String) ? eval(ini_data) : ini_data) +-%> +<%= ERB.new(IO.read('macros/ini_encode_macro.erb'), nil, '-', '_erbout1').result(OpenStruct.new().send(:binding)) -%> diff --git a/templates/test.ini_simple.erb b/templates/test.ini_simple.erb new file mode 100644 index 0000000..ef5e97c --- /dev/null +++ b/templates/test.ini_simple.erb @@ -0,0 +1,12 @@ +# +# This file is managed by Puppet. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +<%- + item = @ini_data || (ini_data.kind_of?(String) ? eval(ini_data) : ini_data) + delimiter = ' ' + section_is_comment = true +-%> +<%= ERB.new(IO.read('macros/ini_encode_macro.erb'), nil, '-', '_erbout1').result(OpenStruct.new().send(:binding)) -%> diff --git a/templates/test.json.erb b/templates/test.json.erb new file mode 100644 index 0000000..9444eaa --- /dev/null +++ b/templates/test.json.erb @@ -0,0 +1,5 @@ +<%- + item = @json_data || (json_data.kind_of?(String) ? eval(json_data) : json_data) + macro_path = 'macros/json_encode_macro.erb' +-%> +<%= ERB.new(IO.read(macro_path), nil, '-', '_erbout1').result(OpenStruct.new().send(:binding)) -%> diff --git a/templates/test.toml.erb b/templates/test.toml.erb new file mode 100644 index 0000000..d02120d --- /dev/null +++ b/templates/test.toml.erb @@ -0,0 +1,11 @@ +# +# This file is managed by Puppet. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +<%- + item = @toml_data || (toml_data.kind_of?(String) ? eval(toml_data) : toml_data) + macro_path = 'macros/toml_encode_macro.erb' +-%> +<%= ERB.new(IO.read(macro_path), nil, '-', '_erbout1').result(OpenStruct.new().send(:binding)) -%> diff --git a/templates/test.xml.erb b/templates/test.xml.erb new file mode 100644 index 0000000..68e7ab9 --- /dev/null +++ b/templates/test.xml.erb @@ -0,0 +1,15 @@ + + + + +<%- + item = @xml_data || (xml_data.kind_of?(String) ? eval(xml_data) : xml_data) + macro_path = 'macros/xml_encode_macro.erb' +-%> +<%= ERB.new(IO.read(macro_path), nil, '-', '_erbout1').result(OpenStruct.new().send(:binding)) -%> diff --git a/templates/test.yaml.erb b/templates/test.yaml.erb new file mode 100644 index 0000000..14d4326 --- /dev/null +++ b/templates/test.yaml.erb @@ -0,0 +1,11 @@ +# +# This file is managed by Puppet. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +<%- + item = @yaml_data || (yaml_data.kind_of?(String) ? eval(yaml_data) : yaml_data) + macro_path = 'macros/yaml_encode_macro.erb' +-%> +<%= ERB.new(IO.read(macro_path), nil, '-', '_erbout1').result(OpenStruct.new().send(:binding)) -%> diff --git a/yaml_converter.py b/yaml_converter.py index 3d2b9e1..416dc0e 100755 --- a/yaml_converter.py +++ b/yaml_converter.py @@ -8,7 +8,7 @@ def parse_arguments(): - description = 'YAML to TOML converter' + description = 'YAML converter' epilog = ( "Examples:\n" " $ %(prog)s -f apache -v apache_data -y ./vars/apache_test.yaml\n" @@ -19,22 +19,22 @@ def parse_arguments(): epilog=epilog, formatter_class=RawTextHelpFormatter) parser.add_argument( - '--format', '-f', + '-f', '--format', metavar='FORMAT', choices=['apache', 'erlang', 'ini', 'json', 'toml', 'xml', 'yaml'], required=True, - help='Output format') + help='output format') parser.add_argument( - '--path', '-p', + '-p', '--path', metavar='PATH', default='.', - help='Path to the macros directory (default: .)') + help='path to the macros directory (default: .)') parser.add_argument( - '--var', '-v', + '-v', '--var', metavar='NAME', - help='Read data from certain YAML variable') + help='read data from certain YAML variable') parser.add_argument( - '--yaml', '-y', + '-y', '--yaml', metavar='FILE', dest='yaml_fh', type=argparse.FileType('r'), @@ -63,7 +63,7 @@ def main(): # Create Jinja2 template object t = Template(template_text) - # Convert the YAML data to INI format + # Convert the YAML data to the final format encode_method = getattr(t.module, '%s_encode' % args.format) output = encode_method(yaml_data) diff --git a/yaml_converter.rb b/yaml_converter.rb new file mode 100755 index 0000000..8513c6f --- /dev/null +++ b/yaml_converter.rb @@ -0,0 +1,112 @@ +#!/usr/bin/env ruby + +require 'erb' +require 'optparse' +require 'ostruct' +require 'yaml' + +def parse_arguments + args = OpenStruct.new + + # Default values + args.format = nil + args.help = false + args.path = '.' + args.var = nil + args.yaml_file = nil + + options = OptionParser.new do |opts| + opts.program_name = $PROGRAM_NAME + opts.banner = "usage: #{opts.program_name} " \ + '[-h] -f FORMAT [-p PATH] [-v NAME] [-y FILE]' + + opts.separator('') + opts.separator('YAML converter') + opts.separator('') + opts.separator('optional arguments:') + + opts.on( + '-f', '--format FORMAT', + 'output format') do |f| + args.format = f + end + + opts.on( + '-h', '--help', + 'show this help message and exit') do |h| + args.help = h + end + + opts.on( + '-p', '--path PATH', + 'path to the macros directory (default: .)') do |p| + args.path = p + end + + opts.on( + '-v', '--var NAME', + 'read data from certain YAML variable') do |v| + args.var = v + end + + opts.on( + '-y', '--yaml FILE', + 'YAML file to convert (default: -)') do |y| + args.yaml_file = y + end + + opts.separator('') + opts.separator('Examples:') + opts.separator(" $ #{opts.program_name} -f apache -v apache_data -y " \ + './vars/apache_test.yaml') + opts.separator(" $ #{opts.program_name} -f json -v json_data -y " \ + './vars/json_test.yaml') + end + + options.parse! + + return args, options +end + +def main + # Parse command line arguments + args, options = parse_arguments + + # Check if help should be displayed + if args.help + print options.help + exit + end + + # Check if format was specified + if args.format.nil? + print options.help + abort("\nERROR: Format not specified.") + end + + # Check if specified format is supported + formats = %w(apache erlang ini json toml xml yaml) + unless formats.include?(args.format) + abort('ERROR: Unsuported format. Suported formats are: ' + + formats.join(', ')) + end + + # Check if the YAML file exists + abort('ERROR: YAML file not found.') unless File.exist?(args.yaml_file) + + if args.var.nil? + item = YAML.load_file(args.yaml_file) + else + item = YAML.load_file(args.yaml_file)[args.var] + end + + # Check if the macro file exists + macro_path = "#{args.path}/macros/#{args.format}_encode_macro.erb" + abort('ERROR: Macro file not found.') unless File.exist?(macro_path) + + # Convert the YAML data to the final format + binding = OpenStruct.new.send(:binding) + print ERB.new(IO.read(macro_path), nil, '-', '_erbout0').result(binding) +end + +main if __FILE__ == $PROGRAM_NAME From d32a0d4032b8c818b52875a246ec350818fc7e37 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Mon, 13 Apr 2015 17:08:10 +0100 Subject: [PATCH 28/32] Support input from pipeline in the yaml_converter.rb --- yaml_converter.rb | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/yaml_converter.rb b/yaml_converter.rb index 8513c6f..2de51b0 100755 --- a/yaml_converter.rb +++ b/yaml_converter.rb @@ -91,13 +91,22 @@ def main formats.join(', ')) end - # Check if the YAML file exists - abort('ERROR: YAML file not found.') unless File.exist?(args.yaml_file) + if args.yaml_file.nil? + # Read data from pipeline + yaml_input = ARGF.read + yaml_load_fnc = YAML.method(:load) + else + # Check if the YAML file exists + abort('ERROR: YAML file not found.') unless File.exist?(args.yaml_file) + yaml_input = args.yaml_file + yaml_load_fnc = YAML.method(:load_file) + end + # Convert the YAML data to the Ruby data structure if args.var.nil? - item = YAML.load_file(args.yaml_file) + item = yaml_load_fnc.call(yaml_input) else - item = YAML.load_file(args.yaml_file)[args.var] + item = yaml_load_fnc.call(yaml_input)[args.var] end # Check if the macro file exists From 98c9b6b602326186a8aa3b263564e6837e283ceb Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Tue, 14 Apr 2015 11:10:44 +0100 Subject: [PATCH 29/32] Adding TOC into the README.md --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 35aaad5..74201d0 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,24 @@ Set of Jinja2 and ERB macros which help to encode Python and Ruby data structure into a different file format. +--- + +* [Motivation](#motivation) +* [Supported formats](#supported-formats) +* Examples + * [Ansible example](#ansible-example) + * [Puppet example](#puppet-example) +* [Macros parameters](#macros-parameters) +* [Limitations](#limitations) +* Programming support + * [Python example](#python-example) + * [Ruby example](#ruby-example) +* [License](#license) +* [Author](#author) + +--- + + Motivation ---------- From 903cb79881296671296823a56bad83a1e3ecdad0 Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Thu, 7 May 2015 17:46:15 +0100 Subject: [PATCH 30/32] Adding Logstash format for Jinja2 --- README.md | 40 ++++++++++ macros/logstash_encode_macro.erb | 4 + macros/logstash_encode_macro.j2 | 126 +++++++++++++++++++++++++++++++ site.pp | 8 ++ site.yaml | 12 +++ templates/test.logstash.erb | 11 +++ templates/test.logstash.j2 | 9 +++ vars/logstash_test.yaml | 33 ++++++++ yaml_converter.py | 2 +- 9 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 macros/logstash_encode_macro.erb create mode 100644 macros/logstash_encode_macro.j2 create mode 100644 templates/test.logstash.erb create mode 100644 templates/test.logstash.j2 create mode 100644 vars/logstash_test.yaml diff --git a/README.md b/README.md index 74201d0..50253ac 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ Supported formats - Erlang config format - INI - JSON +- Logstash - TOML - XML - YAML @@ -369,6 +370,45 @@ following parameter should be addressed by a keyword (e.g. `indent=" "` where > Relative or absolute path to the macro. It's used only in ERB macros. +### `logstash_encode()` + +- `item` + + > Variable holding the input data for the macro. + +- `convert_bools=false` + + > Indicates whether Boolean values presented as a string should be converted + > to a real Booblean value. For example `var1: 'True'` would be represented + > as a string but using the `convert_bools=true` will convert it to a Boolean + > like it would be defined like this: `var1: true`. You can also use the YAML + > type casting by using `!!bool` in front of your value (e.g. + > `!!bool "true"`). + +- `convert_nums=false` + + > Indicates whether number presented as a string should be converted to + > number. For example `var1: '123'` would be represented as a string but + > using the `convert_nums=true` will convert it to a number like it would + > be defined like this: `var1: 123`. You can also use the YAML type casting + > to convert string to number (e.g. `!!int "1234"`, `!!float "3.14"`). + +- `indent=" "` + + > Defines the indentation unit. + +- `level=0` + + > Indicates the initial level of the indentation. Value `0` starts indenting + > from the beginning of the line. Setting the value to higher than `0` + > indents the content by `indent * level`. + +- `prevtype=""` + + > Defines the type of the previous item in the recursive macro calls. It's used + > only internally in the macro. + + ### `toml_encode()` - `item` diff --git a/macros/logstash_encode_macro.erb b/macros/logstash_encode_macro.erb new file mode 100644 index 0000000..299c837 --- /dev/null +++ b/macros/logstash_encode_macro.erb @@ -0,0 +1,4 @@ +<%-# + # ERB macro which converts Ruby data structure to Logstash format + #-%> +TODO diff --git a/macros/logstash_encode_macro.j2 b/macros/logstash_encode_macro.j2 new file mode 100644 index 0000000..e3b863c --- /dev/null +++ b/macros/logstash_encode_macro.j2 @@ -0,0 +1,126 @@ +{# + # Jinja2 macro which converts Python data structure to Logstash format + #} + +{%- macro logstash_encode( + item, + convert_bools=false, + convert_nums=false, + indent=" ", + level=0, + prevtype="") %} + + {%- if item is mapping %} + {#- The item is a dict -#} + + {%- if prevtype in ["value", "value_hash", "array"] -%} + {{ "{\n" }} + {%- endif -%} + + {%- for key, val in item.iteritems() | sort -%} + {%- if key[0] == ":" -%} + {{ indent * level ~ key[1:] ~ " {\n" }}{{ logstash_encode( + val, + convert_bools=convert_bools, + convert_nums=convert_nums, + indent=indent, + level=level+1, + prevtype="block") }} + + {%- if loop.last -%} + {%- if val is string or + val is number or + val in [True, False] or + (val is mapping and val.keys()[0][0] != ":") -%} + {{ "\n" ~ indent * level ~ "}\n" }} + {%- else -%} + {{ indent * level ~ "}\n" }} + {%- endif %} + {%- endif %} + {%- else -%} + {{ indent * level }} + + {%- if prevtype == "value_hash" -%} + {{ '"' ~ key ~ '" => ' }} + {%- else -%} + {{ key ~ " => " }} + {%- endif -%} + + {{ logstash_encode( + val, + convert_bools=convert_bools, + convert_nums=convert_nums, + indent=indent, + level=level+1, + prevtype=("value_hash" if val is mapping else "value")) }} + {%- endif %} + + {%- if not loop.last and + ( + val is string or + val is number or + item in [True, False]) -%} + {{ "\n" }} + {%- endif %} + {%- endfor %} + + {%- if prevtype in ["value", "value_hash", "array"] -%} + {{ "\n" ~ indent * (level-1) ~ "}" }} + + {%- if prevtype in ["value", "value_array"] -%} + {{ "\n" }} + {%- endif %} + {%- endif %} + + {%- elif item is number or + (convert_nums and ( + item | int | string == item or + item | float | string == item)) or + (convert_bools and item in ["true", "True", "false", "False"]) or + item in [True, False] %} + + {#- Item is a value of a number or boolean -#} + + {{ item | lower }} + + {%- elif item is string %} + {#- The item is a string -#} + + {{ '"' ~ item ~ '"' }} + + {%- else %} + {#- The item is a list -#} + + {%- for val in item %} + {%- if val is mapping and val.keys()[0][0] == ":" -%} + {#- value is a block -#} + + {{ logstash_encode( + val, + convert_bools=convert_bools, + convert_nums=convert_nums, + indent=indent, + level=level, + prevtype="block") }} + {%- else %} + {%- if loop.first -%} + {{ "[\n" }} + {%- endif -%} + + {{ indent * level }}{{ logstash_encode( + val, + convert_bools=convert_bools, + convert_nums=convert_nums, + indent=indent, + level=level+1, + prevtype="array") }} + + {%- if loop.last -%} + {{ "\n" ~ indent * (level-1) ~ "]\n" }} + {%- else -%} + {{ ",\n" }} + {%- endif %} + {%- endif %} + {%- endfor -%} + {%- endif %} +{%- endmacro %} diff --git a/site.pp b/site.pp index f3e3970..4a642a3 100644 --- a/site.pp +++ b/site.pp @@ -37,6 +37,14 @@ content => template('test.json.erb'), } + ### Logstash + $json_data = hiera('logstash_data') + + file { '/tmp/test.logstash' : + ensure => present, + content => template('test.logstash.erb'), + } + ### TOML $toml_data = hiera('toml_data') diff --git a/site.yaml b/site.yaml index be37447..8b0c841 100644 --- a/site.yaml +++ b/site.yaml @@ -60,6 +60,18 @@ tags: - json +- name: Logstash encode example + hosts: all + vars_files: + - vars/logstash_test.yaml + tasks: + - name: Create test.logstash file + template: + src: templates/test.logstash.j2 + dest: /tmp/test.logstash + tags: + - logstash + - name: TOML encode example hosts: all vars_files: diff --git a/templates/test.logstash.erb b/templates/test.logstash.erb new file mode 100644 index 0000000..88c13e9 --- /dev/null +++ b/templates/test.logstash.erb @@ -0,0 +1,11 @@ +# +# This file is managed by Puppet. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +<%- + item = @logstash_data || (logstash_data.kind_of?(String) ? eval(logstash_data) : logstash_data) + macro_path = 'macros/logstash_encode_macro.erb' +-%> +<%= ERB.new(IO.read(macro_path), nil, '-', '_erbout1').result(OpenStruct.new().send(:binding)) -%> diff --git a/templates/test.logstash.j2 b/templates/test.logstash.j2 new file mode 100644 index 0000000..f854c68 --- /dev/null +++ b/templates/test.logstash.j2 @@ -0,0 +1,9 @@ +# +# This file is managed by Ansible. +# Do not edit this file manually. +# Any changes will be automatically reverted. +# + +{% from "macros/logstash_encode_macro.j2" import logstash_encode with context -%} + +{{ logstash_encode(logstash_data) }} diff --git a/vars/logstash_test.yaml b/vars/logstash_test.yaml new file mode 100644 index 0000000..a7cb5c4 --- /dev/null +++ b/vars/logstash_test.yaml @@ -0,0 +1,33 @@ +--- + +logstash_data: + - :input: + - :file: + path: /var/log/messages + type: syslog + - :file: + path: /var/log/apache/access.log + type: apache + - :filter: + - ':if [type] == "apache"': + - ':if [status] =~ /^5\d\d/': + :nagios: + codes: plain + workers: 1 + - ':else if [status] =~ /^4\d\d/': + :elasticsearch: + xxx: + aaa: bbb + ccc: ddd + hosts: + - es-server1 + - es-server2 + query: "type:start AND operation:%{[opid]}" + fields: + - "@timestamp" + - started + - :output: + - :file: + path: /var/log/%{type}.%{+yyyy.MM.dd.HH} + - :statsd: + increment: apache.%{[response][status]} diff --git a/yaml_converter.py b/yaml_converter.py index 416dc0e..fbbb086 100755 --- a/yaml_converter.py +++ b/yaml_converter.py @@ -21,7 +21,7 @@ def parse_arguments(): parser.add_argument( '-f', '--format', metavar='FORMAT', - choices=['apache', 'erlang', 'ini', 'json', 'toml', 'xml', 'yaml'], + choices=['apache', 'erlang', 'ini', 'json', 'logstash', 'toml', 'xml', 'yaml'], required=True, help='output format') parser.add_argument( From 624ed05b65743a82bcfdef525176e6cfef5c71ee Mon Sep 17 00:00:00 2001 From: Jiri Tyr Date: Mon, 18 May 2015 16:16:16 +0100 Subject: [PATCH 31/32] Fixing missing newline in the XML macro --- macros/xml_encode_macro.erb | 2 +- macros/xml_encode_macro.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/xml_encode_macro.erb b/macros/xml_encode_macro.erb index 69b82be..be8a534 100644 --- a/macros/xml_encode_macro.erb +++ b/macros/xml_encode_macro.erb @@ -24,7 +24,7 @@ prev_list.delete_at(1) -%> <%- if val == nil -%> - <%--%><%= indent * level + '<' + key.to_s + '/>' -%> + <%--%><%= indent * level + '<' + key.to_s + '/>\n' -%> <%- elsif val.kind_of?(Array) -%> <%-# Only if val is an array -%> <%- diff --git a/macros/xml_encode_macro.j2 b/macros/xml_encode_macro.j2 index 33774ed..11e1dc6 100644 --- a/macros/xml_encode_macro.j2 +++ b/macros/xml_encode_macro.j2 @@ -23,7 +23,7 @@ {%- if prev_list.insert(0, 0) and prev_list.remove(1) %}{% endif %} {%- if val is none -%} - {{ indent * level }}<{{ key }} /> + {{ indent * level ~ "<" ~ key ~ " />\n" }} {%- elif val is sequence and val is not string and val is not mapping -%} {#- Only if val is a list -#} {{ xml_encode( From e4373425868eb02f7d3474724707b3e53d29b9dc Mon Sep 17 00:00:00 2001 From: Jason Royle Date: Wed, 15 Mar 2017 14:28:22 +0000 Subject: [PATCH 32/32] Removed the config encode macros submodule Submodules are incompatible with Ansible Galaxy imports --- .gitmodules | 3 --- templates/config-encoder-macros | 1 - 2 files changed, 4 deletions(-) delete mode 160000 templates/config-encoder-macros diff --git a/.gitmodules b/.gitmodules index a9a300c..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "templates/config-encoder-macros"] - path = templates/config-encoder-macros - url = https://github.com/picotrading/config-encoder-macros.git diff --git a/templates/config-encoder-macros b/templates/config-encoder-macros deleted file mode 160000 index 624ed05..0000000 --- a/templates/config-encoder-macros +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 624ed05b65743a82bcfdef525176e6cfef5c71ee