From d8658dbcf7f05c2058154f95118c2be19bbadef0 Mon Sep 17 00:00:00 2001 From: Jatin Arora Date: Mon, 9 Sep 2024 14:29:24 +0530 Subject: [PATCH 1/4] feat: add script to parse swagger generate from ams This commit adds the script to parse the swagger generated via ams to render the node and ms configuration fields. --- scripts/parse.py | 80 ++++++++++++++++++++++++++++++++++++++++ scripts/requirements.txt | 2 + scripts/template.md.j2 | 45 ++++++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100755 scripts/parse.py create mode 100644 scripts/requirements.txt create mode 100644 scripts/template.md.j2 diff --git a/scripts/parse.py b/scripts/parse.py new file mode 100755 index 00000000..41406c7b --- /dev/null +++ b/scripts/parse.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 + +import argparse +import json + +from jinja2 import Environment +from jinja2.loaders import FileSystemLoader + + +def main() -> int: + parser = argparse.ArgumentParser( + prog="AMS swagger parser", + description="Extracts AMS configuration data", + ) + parser.add_argument( + "swagger_path", + help="Path to the swagger json file (Should conform to swagger spec 2.0)", + ) + parser.add_argument( + "--output", + dest="output_file", + help="Destination of the rendered configuration file", + default="ams-configuration.md", + ) + args = parser.parse_args() + with open(args.swagger_path, mode="r") as f: + swagger = json.load(f) + + configs = _parse_config_schema(swagger) + nodes = _parse_node_schema(swagger) + env = Environment(loader=FileSystemLoader(".")) + templ = env.get_template("template.md.j2") + text = templ.render(configs=configs, nodes=nodes) + with open(args.output_file, mode="w+") as op: + op.write(text) + + +def _parse_config_schema(swagger) -> dict: + ams_config_path = "/1.0/config" + config_props = swagger["paths"][ams_config_path]["get"]["responses"]["200"][ + "schema" + ]["properties"]["metadata"]["properties"]["config"]["properties"] + for prop in config_props.values(): + if "description" in prop: + prop["description"] = prop["description"].replace("\n", " ") + return config_props + + +def _parse_node_schema(swagger) -> dict: + node_props = swagger["definitions"]["NodePatch"]["properties"] + for base, prop in dict(node_props).items(): + key = "" + if prop["type"] == "object": + key = "additionalProperties" + if prop["type"] == "array": + key = "items" + if key: + if schema_name := prop[key].get("$ref"): + name = schema_name.split("/")[-1] + if schema := swagger["definitions"][name]: + for subprop_name, subprop_attrs in schema["properties"].items(): + if subprop_name == "id": + continue + new_base = base + if base == "gpus": + new_base = f"{base}." + node_props[f"{new_base}.{subprop_name.replace('_', '-')}"] = ( + subprop_attrs + ) + node_props.pop(base) + else: + val = node_props.pop(base) + node_props[base.replace("_", "-")] = val + if "description" in prop: + prop["description"] = prop["description"].replace("\n", " ") + return node_props + + +if __name__ == "__main__": + SystemExit(main()) diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 00000000..f5617e6d --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,2 @@ +Jinja2==3.1.4 +MarkupSafe==2.1.5 diff --git a/scripts/template.md.j2 b/scripts/template.md.j2 new file mode 100644 index 00000000..8574659b --- /dev/null +++ b/scripts/template.md.j2 @@ -0,0 +1,45 @@ +(ref-ams-configuration)= +# AMS configuration + +The Anbox Management Service (AMS) provides various configuration items to customise its behaviour. The following table lists the available configuration items and their meaning. + +| Name
*(Type, Default)* | Description | +|-----|-----------------------| +{%- for key, config in configs.items() %} +| `{{ key }}`
*({{ config.type }}, {{ config.default or "N/A"}})* | {{ config.description }} | +{%- endfor %} + +(sec-node-configuration)= +## Node-specific configuration + +In a cluster setup, there are configuration items that can be customised for each node. The following table lists the available configuration items and their meaning. + +| Name
*(Type, Default)* | Description | +|--------------------------|-------------------------| +{%- for key, node in nodes.items() %} +| `{{ key }}`
*({{ node.type }}, {{ node.default or "N/A"}})* | {{ node.description }} | +{%- endfor %} + +See {ref}`howto-configure-cluster-nodes` for instructions on how to set these configuration items. + +## Objects managed by AMS + +AMS manages various objects such as applications, images, instances, nodes and addons. + +The object names must adhere to the following criteria: + +* Minimum character limit: 3 +* Maximum character limit: 255 +* Can contain: + - Alphabets (a-z, A-Z) + - Numbers (0-9) + - Allowed special characters: `-` (hyphen), `_` (underscore), `:` (colon), `.` (period). + +When you create an instance, the same criteria apply to the following options as well: + +* `boot_activity` +* `platform` +* `boot_package` + +The object ids are generated by AMS and have a length of 20 characters. + From f0e36d5c401c1cf59425f8d9141fbe0b0c0cb40d Mon Sep 17 00:00:00 2001 From: Jatin Arora Date: Mon, 9 Sep 2024 15:40:22 +0530 Subject: [PATCH 2/4] feat: fetch swagger from url This commit adds the hosted swagger url for ams and fetches it as a default from there in order to parse it. --- scripts/parse.py | 19 ++++++++++++++++--- scripts/requirements.txt | 5 +++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/scripts/parse.py b/scripts/parse.py index 41406c7b..0fffd6b4 100755 --- a/scripts/parse.py +++ b/scripts/parse.py @@ -3,9 +3,14 @@ import argparse import json +import requests from jinja2 import Environment from jinja2.loaders import FileSystemLoader +SWAGGER_URL = ( + "https://canonical.github.io/anbox-cloud.github.com/latest/ams/swagger.json" +) + def main() -> int: parser = argparse.ArgumentParser( @@ -13,18 +18,26 @@ def main() -> int: description="Extracts AMS configuration data", ) parser.add_argument( - "swagger_path", + "-i", + "--input-file", + dest="swagger_path", + required=False, help="Path to the swagger json file (Should conform to swagger spec 2.0)", ) parser.add_argument( + "-o", "--output", dest="output_file", help="Destination of the rendered configuration file", default="ams-configuration.md", ) args = parser.parse_args() - with open(args.swagger_path, mode="r") as f: - swagger = json.load(f) + if args.swagger_path: + with open(args.swagger_path, mode="r") as f: + swagger = json.load(f) + else: + response = requests.get(SWAGGER_URL) + swagger = response.json() configs = _parse_config_schema(swagger) nodes = _parse_node_schema(swagger) diff --git a/scripts/requirements.txt b/scripts/requirements.txt index f5617e6d..c5395fb2 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,2 +1,7 @@ +certifi==2024.8.30 +charset-normalizer==3.3.2 +idna==3.8 Jinja2==3.1.4 MarkupSafe==2.1.5 +requests==2.32.3 +urllib3==2.2.2 From 974684a6fb20e37996697c7853e9157bb13743a9 Mon Sep 17 00:00:00 2001 From: Jatin Arora Date: Mon, 9 Sep 2024 16:53:31 +0530 Subject: [PATCH 3/4] feat: add ams configuration generation at build time This commit modifies the configuration in sphinx to generate ams configuration at build and inject it as well. --- build_requirements.py | 2 +- conf.py | 4 ++++ custom_conf.py | 17 +++++++++++++++-- ...{template.md.j2 => ams-configuration.md.j2} | 0 scripts/{parse.py => ams_configuration.py} | 18 +++++++++++++----- 5 files changed, 33 insertions(+), 8 deletions(-) rename scripts/{template.md.j2 => ams-configuration.md.j2} (100%) rename scripts/{parse.py => ams_configuration.py} (87%) diff --git a/build_requirements.py b/build_requirements.py index e1ccb4ac..e45f0815 100644 --- a/build_requirements.py +++ b/build_requirements.py @@ -1,6 +1,6 @@ import sys -sys.path.append('./') +sys.path.append("./") from custom_conf import * # The file contains helper functions and the mechanism to build the diff --git a/conf.py b/conf.py index dfbd56d0..6e0bf2fe 100644 --- a/conf.py +++ b/conf.py @@ -133,3 +133,7 @@ if 'github_issues' in html_context and html_context['github_issues'] and not disable_feedback_button: html_js_files.append('github_issue_links.js') html_js_files.extend(custom_html_js_files) + +# Anbox specific function to generate dynamic AMS configuration +# Add this change to conf.py every time the starter pack is upgraded to a later version. +generate_ams_configuration() diff --git a/custom_conf.py b/custom_conf.py index e900d6fa..7ce0833e 100644 --- a/custom_conf.py +++ b/custom_conf.py @@ -210,7 +210,20 @@ ## Add any configuration that is not covered by the common conf.py file. # Define a :center: role that can be used to center the content of table cells. -rst_prolog = ''' +rst_prolog = """ .. role:: center :class: align-center -''' +""" + + +## Generate dynamic configuration using scripts +# Inject AMS configuration valuues and Node configuration values from the swagger +# specification hosted on Github. +def generate_ams_configuration(): + from scripts.ams_configuration import get_swagger_from_url, parse_swagger + + with open("scripts/requirements.txt", "r") as f: + for req in f.readlines(): + custom_required_modules.append(req) + ams_configuration_file = "reference/ams-configuration.md" + parse_swagger(get_swagger_from_url(), ams_configuration_file) diff --git a/scripts/template.md.j2 b/scripts/ams-configuration.md.j2 similarity index 100% rename from scripts/template.md.j2 rename to scripts/ams-configuration.md.j2 diff --git a/scripts/parse.py b/scripts/ams_configuration.py similarity index 87% rename from scripts/parse.py rename to scripts/ams_configuration.py index 0fffd6b4..3d709b68 100755 --- a/scripts/parse.py +++ b/scripts/ams_configuration.py @@ -2,6 +2,7 @@ import argparse import json +from typing import Dict import requests from jinja2 import Environment @@ -29,22 +30,29 @@ def main() -> int: "--output", dest="output_file", help="Destination of the rendered configuration file", - default="ams-configuration.md", + default="reference/ams-configuration.md", ) args = parser.parse_args() if args.swagger_path: with open(args.swagger_path, mode="r") as f: swagger = json.load(f) else: - response = requests.get(SWAGGER_URL) - swagger = response.json() + swagger = get_swagger_from_url() + parse_swagger(swagger, args.output_file) + +def get_swagger_from_url() -> Dict: + response = requests.get(SWAGGER_URL) + return response.json() + + +def parse_swagger(swagger, output_file): configs = _parse_config_schema(swagger) nodes = _parse_node_schema(swagger) env = Environment(loader=FileSystemLoader(".")) - templ = env.get_template("template.md.j2") + templ = env.get_template("scripts/ams-configuration.md.j2") text = templ.render(configs=configs, nodes=nodes) - with open(args.output_file, mode="w+") as op: + with open(output_file, mode="w+") as op: op.write(text) From 2e12afae9bfb3b1a3e0cc24f3b9719329cfae0f5 Mon Sep 17 00:00:00 2001 From: Jatin Arora Date: Mon, 23 Sep 2024 18:56:49 +0530 Subject: [PATCH 4/4] feat(scripts): add deprecation and doc hyperlinks in the script This commit adds the feature to insert deprecated fields and other hyperlinks from the swagger. --- scripts/ams-configuration.md.j2 | 4 ++-- scripts/ams_configuration.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/ams-configuration.md.j2 b/scripts/ams-configuration.md.j2 index 8574659b..8bc56d16 100644 --- a/scripts/ams-configuration.md.j2 +++ b/scripts/ams-configuration.md.j2 @@ -6,7 +6,7 @@ The Anbox Management Service (AMS) provides various configuration items to custo | Name
*(Type, Default)* | Description | |-----|-----------------------| {%- for key, config in configs.items() %} -| `{{ key }}`
*({{ config.type }}, {{ config.default or "N/A"}})* | {{ config.description }} | +| `{{ key }}`
*({{ config.type }}, {{ config.default or "N/A"}})*{% if config.x_deprecated_since %}
*(Deprecated in {{ config.x_deprecated_since }})*{% endif %} | {{ config.description }}{% if config.x_docs_ref %} See {ref}`{{ config.x_docs_ref }}`. {% endif %}| {%- endfor %} (sec-node-configuration)= @@ -17,7 +17,7 @@ In a cluster setup, there are configuration items that can be customised for eac | Name
*(Type, Default)* | Description | |--------------------------|-------------------------| {%- for key, node in nodes.items() %} -| `{{ key }}`
*({{ node.type }}, {{ node.default or "N/A"}})* | {{ node.description }} | +| `{{ key }}`
*({{ node.type }}, {{ node.default or "N/A"}})*{% if node.x_deprecated_since %}
*(Deprecated in {{ node.x_deprecated_since }})*{% endif %} | {{ node.description }} {% if node.x_docs_ref %} See {ref}`{{ node.x_docs_ref }}`. {% endif %}| {%- endfor %} See {ref}`howto-configure-cluster-nodes` for instructions on how to set these configuration items. diff --git a/scripts/ams_configuration.py b/scripts/ams_configuration.py index 3d709b68..1544ca97 100755 --- a/scripts/ams_configuration.py +++ b/scripts/ams_configuration.py @@ -46,9 +46,21 @@ def get_swagger_from_url() -> Dict: return response.json() +def insert_custom_fields(props: Dict): + for prop in props.values(): + val = prop.pop("x-docs-ref", "") + if val: + prop["x_docs_ref"] = val + val = prop.pop("x-deprecated-since", "") + if val: + prop["x_deprecated_since"] = val + + def parse_swagger(swagger, output_file): configs = _parse_config_schema(swagger) nodes = _parse_node_schema(swagger) + insert_custom_fields(configs) + insert_custom_fields(nodes) env = Environment(loader=FileSystemLoader(".")) templ = env.get_template("scripts/ams-configuration.md.j2") text = templ.render(configs=configs, nodes=nodes)