From 9f5230587d117ccf96272eaca30e1a81f3e051f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20Roch=C3=A9?= Date: Tue, 27 Feb 2024 12:24:15 +0100 Subject: [PATCH 1/6] jsonschema: support definitions as an alias for $defs --- lib/generator.ml | 18 ++++++++++-------- lib/json_schema.atd | 1 + lib/utils.ml | 4 ++++ tests/mocks/jsonchema_definitions.json | 23 +++++++++++++++++++++++ tests/smoke.t | 13 +++++++++++++ 5 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 tests/mocks/jsonchema_definitions.json diff --git a/lib/generator.ml b/lib/generator.ml index df097b8..9c524c3 100644 --- a/lib/generator.ml +++ b/lib/generator.ml @@ -19,19 +19,21 @@ let get_ref_name ref = (* OpenAPI defs *) | [ "#"; "components"; "schemas"; type_name ] -> type_name (* JSON Schema defs *) - | [ "#"; "$defs"; type_name ] -> type_name + | [ "#"; ("$defs" | "definitions"); type_name ] -> type_name | _ -> failwith - (Printf.sprintf "Unsupported ref value: %s. Supported ref URI are: #/components/schemas/* and #/$defs/*" ref) + (Printf.sprintf + "Unsupported ref value: %s. Supported ref URI are: #/components/schemas/* and #/$defs/* and #/definitions/*" + ref + ) let output = Buffer.create 16 let input_toplevel_schemas = ref [] let get_schema_by_ref ~schema ref = let defs = - match schema.defs with - | None -> !input_toplevel_schemas - | Some defs -> defs @ !input_toplevel_schemas + let defs = List.concat_map Utils.list_of_nonempty [ schema.defs; schema.definitions ] in + defs @ !input_toplevel_schemas in List.find_map (function @@ -107,6 +109,7 @@ let merge_all_of schema = dependent_required = merge_lists (fun schema -> schema.dependent_required); format = take_first_opt (fun schema -> schema.format); defs = merge_opt_lists (fun schema -> schema.defs); + definitions = merge_opt_lists (fun schema -> schema.definitions); title = take_first_opt (fun schema -> schema.title); typ = take_first_opt (fun schema -> schema.typ); description = take_first_opt (fun schema -> schema.description); @@ -222,9 +225,8 @@ let make_atd_of_jsonschema input = let schema = Json_schema_j.schema_of_string input in let root_type_name = Option.value ~default:"root" schema.title in let defs = - match schema.defs with - | None -> [] - | Some defs -> List.map (fun (name, schema) -> name, Obj schema) defs + let defs = List.concat_map Utils.list_of_nonempty [ schema.defs; schema.definitions ] in + List.map (fun (name, schema) -> name, Obj schema) defs in make_atd_of_schemas ([ root_type_name, Obj schema ] @ defs) diff --git a/lib/json_schema.atd b/lib/json_schema.atd index 82b5095..c823111 100644 --- a/lib/json_schema.atd +++ b/lib/json_schema.atd @@ -88,6 +88,7 @@ type schema = { (* 8.2.4. re-usable JSON Schemas *) ~defs : (string * schema) list nullable; + ~definitions: (string * schema) list nullable; (* 9. basic metadata annotations *) ~title : string nullable; diff --git a/lib/utils.ml b/lib/utils.ml index 9c80728..37a639e 100644 --- a/lib/utils.ml +++ b/lib/utils.ml @@ -98,3 +98,7 @@ let shortest_list lists = lists |> List.sort (fun a b -> compare (List.length a) let nonempty_list_opt = function | [] -> None | non_empty_list -> Some non_empty_list + +let list_of_nonempty = function + | None -> [] + | Some l -> l diff --git a/tests/mocks/jsonchema_definitions.json b/tests/mocks/jsonchema_definitions.json new file mode 100644 index 0000000..b5b47af --- /dev/null +++ b/tests/mocks/jsonchema_definitions.json @@ -0,0 +1,23 @@ +{ + "$id": "https://example.com/schemas/customer", + "type": "object", + "properties": { + "first_name": { + "$ref": "#/$defs/name" + }, + "last_name": { + "$ref": "#/definitions/name" + } + }, + "required": [ + "first_name", + "last_name", + "shipping_address", + "billing_address" + ], + "definitions": { + "name": { + "type": "string" + } + } +} diff --git a/tests/smoke.t b/tests/smoke.t index c99866e..30d6344 100644 --- a/tests/smoke.t +++ b/tests/smoke.t @@ -56,3 +56,16 @@ Generate ATD out of JSON Schema that contains defs first_name: name; last_name: name; } + +Generate ATD out of JSON Schema that contains definitions (legacy support) + $ jsonschema2atd --format=jsonschema ./mocks/jsonchema_definitions.json + (* Generated by jsonschema2atd *) + type json = abstract + type int64 = int + + type name = string + + type root = { + first_name: name; + last_name: name; + } From 659a897137fca419d21883888e0c6be948085caf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20Roch=C3=A9?= Date: Tue, 27 Feb 2024 12:52:07 +0100 Subject: [PATCH 2/6] jsonschema: support more string formats for date and email --- lib/json_schema.atd | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/json_schema.atd b/lib/json_schema.atd index c823111..1ddd5bf 100644 --- a/lib/json_schema.atd +++ b/lib/json_schema.atd @@ -27,6 +27,10 @@ type number_format = [ type str_format = [ | Date | Datetime + | Time + | Duration + | Email + | Idn_email ] type format = [ From 5d63791f945d4faf71b9eb3109079886d7e0d637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20Roch=C3=A9?= Date: Tue, 27 Feb 2024 14:48:16 +0100 Subject: [PATCH 3/6] turn description into atd doc --- lib/generator.ml | 35 ++- tests/grok.t | 560 +++++++++++++++++++++++------------------------ tests/one_of.ml | 3 +- tests/smoke.t | 36 +-- 4 files changed, 329 insertions(+), 305 deletions(-) diff --git a/lib/generator.ml b/lib/generator.ml index 9c524c3..ec081bc 100644 --- a/lib/generator.ml +++ b/lib/generator.ml @@ -6,7 +6,15 @@ let record_field_name str = let cleaned_field_name = Utils.sanitize_name str in if String.equal str cleaned_field_name then str else sprintf {|%s |} cleaned_field_name str -let define_type name type_ = sprintf "type %s = %s\n" (type_name name) type_ +let doc_annotation text = sprintf {||} text + +let define_type ~doc ~name ~type_ = + let doc = + match doc with + | None -> "" + | Some doc -> doc_annotation doc + in + sprintf "type %s = %s %s\n" (type_name name) type_ doc let process_int_type schema = match schema.format with @@ -147,7 +155,9 @@ and process_nested_schema_type ~ancestors schema = match merge_all_of schema with | { one_of = Some _; _ } | { typ = Some Object; properties = Some _; _ } | { enum = Some _; _ } -> let nested_type_name = concat_camelCase (List.rev ancestors) in - let nested = define_type nested_type_name (process_schema_type ~ancestors schema) in + let nested = + define_type ~name:nested_type_name ~type_:(process_schema_type ~ancestors schema) ~doc:schema.description + in Buffer.add_string output (nested ^ "\n"); type_name nested_type_name | _ -> process_schema_type ~ancestors schema @@ -157,11 +167,19 @@ and process_object_type ~ancestors schema = let make_record_field (field_name, schema_or_ref) = let type_ = make_type_from_schema_or_ref ~ancestors:(field_name :: ancestors) schema_or_ref in let record_field_name = record_field_name field_name in + let doc = + let content = + match schema_or_ref with + | Ref _ -> None + | Obj schema -> schema.description + in + Option.map doc_annotation content |> Option.value ~default:"" + in match schema_or_ref, is_required field_name with | Obj { default = Some default; enum; _ }, _ -> - sprintf " ~%s : %s;" record_field_name (make_atd_default_value enum default) type_ - | _, true -> sprintf " %s: %s;" record_field_name type_ - | _, false -> sprintf " ?%s: %s option;" record_field_name type_ + sprintf " ~%s %s : %s;" record_field_name doc (make_atd_default_value enum default) type_ + | _, true -> sprintf " %s %s: %s;" record_field_name doc type_ + | _, false -> sprintf " ?%s %s: %s option;" record_field_name doc type_ in match schema.properties with | Some properties -> sprintf "{\n%s\n}" (properties |> List.map make_record_field |> String.concat "\n") @@ -198,7 +216,12 @@ and process_enums enums = let process_schemas (schemas : (string * schema or_ref) list) = List.fold_left (fun acc (name, schema_or_ref) -> - define_type name (make_type_from_schema_or_ref ~ancestors:[ name ] schema_or_ref) :: acc + let doc = + match schema_or_ref with + | Ref _ -> None + | Obj schema -> schema.description + in + define_type ~doc ~name ~type_:(make_type_from_schema_or_ref ~ancestors:[ name ] schema_or_ref) :: acc ) [] schemas diff --git a/tests/grok.t b/tests/grok.t index d4b4ccf..951de8e 100644 --- a/tests/grok.t +++ b/tests/grok.t @@ -5,185 +5,185 @@ Generate ATD types from grok (Grafana Object Development Kit) dashboard types type int64 = int type fieldConfigSourceOverrides = { - matcher: matcherConfig; - properties: dynamicConfigValue list; - } + matcher : matcherConfig; + properties : dynamicConfigValue list; + } type graphPanelType = [ | Graph - ] + ] type graphPanelLegend = { - ~show : bool; - ?sort: string option; - ?sortDesc: bool option; - } + ~show : bool; + ?sort : string option; + ?sortDesc : bool option; + } type heatmapPanelType = [ | Heatmap - ] + ] type panelRepeatDirection = [ | H | V - ] + ] type rangeMapType = [ | Range - ] + ] type rangeMapOptions = { - from: float; - to_ : float; - result: valueMappingResult; - } + from : float; + to_ : float; + result : valueMappingResult; + } type regexMapType = [ | Regex - ] + ] type regexMapOptions = { - pattern: string; - result: valueMappingResult; - } + pattern : string; + result : valueMappingResult; + } type rowPanelType = [ | Row - ] + ] type rowPanelPanels = [ | Panel of panel | GraphPanel of graphPanel | HeatmapPanel of heatmapPanel - ] + ] type specialValueMapType = [ | Special - ] + ] type specialValueMapOptions = { - match_ : specialValueMatch; - result: valueMappingResult; - } + match_ : specialValueMatch; + result : valueMappingResult; + } type valueMapType = [ | Value - ] + ] type variableModelQuery = [ | String of string | Json of json - ] + ] type variableOptionText = [ | String of string | StringList of string list - ] + ] type variableOptionValue = [ | String of string | StringList of string list - ] + ] type dashboardMetadata = { - updateTimestamp: string; - createdBy: string; - updatedBy: string; - extraFields: json; - uid: string; - creationTimestamp: string; - ?deletionTimestamp: string option; - finalizers: string list; - resourceVersion: string; - labels: json; - } + updateTimestamp : string; + createdBy : string; + updatedBy : string; + extraFields : json; + uid : string; + creationTimestamp : string; + ?deletionTimestamp : string option; + finalizers : string list; + resourceVersion : string; + labels : json; + } type dashboardSpecTime = { - ~from : string; - ~to_ : string; - } + ~from : string; + ~to_ : string; + } type dashboardSpecTimepicker = { - ~hidden : bool; - ~refresh_intervals : string list; - ~collapse : bool; - ~time_options : string list; - } + ~hidden : bool; + ~refresh_intervals : string list; + ~collapse : bool; + ~time_options : string list; + } type dashboardSpecRefresh = [ | Bool of bool | String of string - ] + ] type dashboardSpecPanels = [ | Panel of panel | RowPanel of rowPanel | GraphPanel of graphPanel | HeatmapPanel of heatmapPanel - ] + ] type dashboardSpecTemplating = { - ?list: variableModel list option; - } + ?list : variableModel list option; + } type dashboardSpec = { - ?id: int option; - ?uid: string option; - ?title: string option; - ?description: string option; - ?revision: int64 option; - ?gnetId: string option; - ?tags: string list option; - ~timezone : string; - ~editable : bool; - ?graphTooltip: dashboardCursorSync option; - ?time: dashboardSpecTime option; - ?timepicker: dashboardSpecTimepicker option; - ~fiscalYearStartMonth : int; - ?liveNow: bool option; - ?weekStart: string option; - ?refresh: dashboardSpecRefresh option; - ~schemaVersion : int; - ?version: int option; - ?panels: dashboardSpecPanels list option; - ?templating: dashboardSpecTemplating option; - ?annotations: annotationContainer option; - ?links: dashboardLink list option; - ?snapshot: snapshot option; - } + ?id : int option; + ?uid : string option; + ?title : string option; + ?description : string option; + ?revision : int64 option; + ?gnetId : string option; + ?tags : string list option; + ~timezone : string; + ~editable : bool; + ?graphTooltip : dashboardCursorSync option; + ?time : dashboardSpecTime option; + ?timepicker : dashboardSpecTimepicker option; + ~fiscalYearStartMonth : int; + ?liveNow : bool option; + ?weekStart : string option; + ?refresh : dashboardSpecRefresh option; + ~schemaVersion : int; + ?version : int option; + ?panels : dashboardSpecPanels list option; + ?templating : dashboardSpecTemplating option; + ?annotations : annotationContainer option; + ?links : dashboardLink list option; + ?snapshot : snapshot option; + } type dashboardStatus = { - ?operatorStates: json option; - ?additionalFields: json option; - } + ?operatorStates : json option; + ?additionalFields : json option; + } type statusOperatorStateState = [ | Success | In_progress | Failed - ] + ] type statusOperatorState = { - lastEvaluation: string; - state: statusOperatorStateState; - ?descriptiveState: string option; - ?details: json option; - } + lastEvaluation : string; + state : statusOperatorStateState; + ?descriptiveState : string option; + ?details : json option; + } type dashboard = { - metadata: dashboardMetadata; - spec: dashboardSpec; - status: dashboardStatus; - } + metadata : dashboardMetadata; + spec : dashboardSpec; + status : dashboardStatus; + } type _kubeObjectMetadata = { - uid: string; - creationTimestamp: string; - ?deletionTimestamp: string option; - finalizers: string list; - resourceVersion: string; - labels: json; - } + uid : string; + creationTimestamp : string; + ?deletionTimestamp : string option; + finalizers : string list; + resourceVersion : string; + labels : json; + } type variableType = [ | Query @@ -194,71 +194,71 @@ Generate ATD types from grok (Grafana Object Development Kit) dashboard types | Textbox | Custom | System - ] + ] - type variableSort = int + type variableSort = int - type variableRefresh = int + type variableRefresh = int type variableOption = { - ?selected: bool option; - text: variableOptionText; - value: variableOptionValue; - } + ?selected : bool option; + text : variableOptionText; + value : variableOptionValue; + } type variableModel = { - type_ : variableType; - name: string; - ?label: string option; - ?hide: variableHide option; - ~skipUrlSync : bool; - ?description: string option; - ?query: variableModelQuery option; - ?datasource: dataSourceRef option; - ?current: variableOption option; - ~multi : bool; - ?options: variableOption list option; - ?refresh: variableRefresh option; - ?sort: variableSort option; - } - - type variableHide = int + type_ : variableType; + name : string; + ?label : string option; + ?hide : variableHide option; + ~skipUrlSync : bool; + ?description : string option; + ?query : variableModelQuery option; + ?datasource : dataSourceRef option; + ?current : variableOption option; + ~multi : bool; + ?options : variableOption list option; + ?refresh : variableRefresh option; + ?sort : variableSort option; + } + + type variableHide = int type valueMappingResult = { - ?text: string option; - ?color: string option; - ?icon: string option; - ?index: int option; - } + ?text : string option; + ?color : string option; + ?icon : string option; + ?index : int option; + } type valueMapping = [ | ValueMap of valueMap | RangeMap of rangeMap | RegexMap of regexMap | SpecialValueMap of specialValueMap - ] + ] type valueMap = { - type_ : valueMapType; - options: json; - } + type_ : valueMapType; + options : json; + } type thresholdsMode = [ | Absolute | Percentage - ] + ] type thresholdsConfig = { - mode: thresholdsMode; - steps: threshold list; - } + mode : thresholdsMode; + steps : threshold list; + } type threshold = { - value: float; - color: string; - } + value : float; + color : string; + } - type target = json + type target = json type specialValueMatch = [ | True_ @@ -267,137 +267,137 @@ Generate ATD types from grok (Grafana Object Development Kit) dashboard types | Nan | Nullnan | Empty - ] + ] type specialValueMap = { - type_ : specialValueMapType; - options: specialValueMapOptions; - } + type_ : specialValueMapType; + options : specialValueMapOptions; + } type snapshot = { - created: string; - expires: string; - external_ : bool; - externalUrl: string; - id: int; - key: string; - name: string; - orgId: int; - updated: string; - ?url: string option; - userId: int; - } + created : string; + expires : string; + external_ : bool; + externalUrl : string; + id : int; + key : string; + name : string; + orgId : int; + updated : string; + ?url : string option; + userId : int; + } type rowPanel = { - type_ : rowPanelType; - ~collapsed : bool; - ?title: string option; - ?datasource: dataSourceRef option; - ?gridPos: gridPos option; - id: int; - panels: rowPanelPanels list; - ?repeat: string option; - } + type_ : rowPanelType; + ~collapsed : bool; + ?title : string option; + ?datasource : dataSourceRef option; + ?gridPos : gridPos option; + id : int; + panels : rowPanelPanels list; + ?repeat : string option; + } type regexMap = { - type_ : regexMapType; - options: regexMapOptions; - } + type_ : regexMapType; + options : regexMapOptions; + } type rangeMap = { - type_ : rangeMapType; - options: rangeMapOptions; - } + type_ : rangeMapType; + options : rangeMapOptions; + } type panel = { - type_ : string; - ?id: int option; - ?pluginVersion: string option; - ?tags: string list option; - ?targets: target list option; - ?title: string option; - ?description: string option; - ~transparent : bool; - ?datasource: dataSourceRef option; - ?gridPos: gridPos option; - ?links: dashboardLink list option; - ?repeat: string option; - ~repeatDirection : panelRepeatDirection; - ?maxPerRow: float option; - ?maxDataPoints: float option; - ?transformations: dataTransformerConfig list option; - ?interval: string option; - ?timeFrom: string option; - ?timeShift: string option; - ?hideTimeOverride: bool option; - ?libraryPanel: libraryPanelRef option; - ?options: json option; - ?fieldConfig: fieldConfigSource option; - } + type_ : string; + ?id : int option; + ?pluginVersion : string option; + ?tags : string list option; + ?targets : target list option; + ?title : string option; + ?description : string option; + ~transparent : bool; + ?datasource : dataSourceRef option; + ?gridPos : gridPos option; + ?links : dashboardLink list option; + ?repeat : string option; + ~repeatDirection : panelRepeatDirection; + ?maxPerRow : float option; + ?maxDataPoints : float option; + ?transformations : dataTransformerConfig list option; + ?interval : string option; + ?timeFrom : string option; + ?timeShift : string option; + ?hideTimeOverride : bool option; + ?libraryPanel : libraryPanelRef option; + ?options : json option; + ?fieldConfig : fieldConfigSource option; + } type matcherConfig = { - ~id : string; - ?options: json option; - } + ~id : string; + ?options : json option; + } type mappingType = [ | Value | Range | Regex | Special - ] + ] type libraryPanelRef = { - name: string; - uid: string; - } + name : string; + uid : string; + } type heatmapPanel = { - type_ : heatmapPanelType; - } + type_ : heatmapPanelType; + } type gridPos = { - ~h : int; - ~w : int; - ~x : int; - ~y : int; - ?static: bool option; - } + ~h : int; + ~w : int; + ~x : int; + ~y : int; + ?static : bool option; + } type graphPanel = { - type_ : graphPanelType; - ?legend: graphPanelLegend option; - } + type_ : graphPanelType; + ?legend : graphPanelLegend option; + } type fieldConfigSource = { - defaults: fieldConfig; - overrides: fieldConfigSourceOverrides list; - } + defaults : fieldConfig; + overrides : fieldConfigSourceOverrides list; + } type fieldConfig = { - ?displayName: string option; - ?displayNameFromDS: string option; - ?description: string option; - ?path: string option; - ?writeable: bool option; - ?filterable: bool option; - ?unit: string option; - ?decimals: float option; - ?min: float option; - ?max: float option; - ?mappings: valueMapping list option; - ?thresholds: thresholdsConfig option; - ?color: fieldColor option; - ?links: json list option; - ?noValue: string option; - ?custom: json option; - } + ?displayName : string option; + ?displayNameFromDS : string option; + ?description : string option; + ?path : string option; + ?writeable : bool option; + ?filterable : bool option; + ?unit : string option; + ?decimals : float option; + ?min : float option; + ?max : float option; + ?mappings : valueMapping list option; + ?thresholds : thresholdsConfig option; + ?color : fieldColor option; + ?links : json list option; + ?noValue : string option; + ?custom : json option; + } type fieldColorSeriesByMode = [ | Min | Max | Last - ] + ] type fieldColorModeId = [ | Thresholds @@ -415,75 +415,75 @@ Generate ATD types from grok (Grafana Object Development Kit) dashboard types | Continuouspurples | Fixed | Shades - ] + ] type fieldColor = { - mode: fieldColorModeId; - ?fixedColor: string option; - ?seriesBy: fieldColorSeriesByMode option; - } + mode : fieldColorModeId; + ?fixedColor : string option; + ?seriesBy : fieldColorSeriesByMode option; + } type dynamicConfigValue = { - ~id : string; - ?value: json option; - } + ~id : string; + ?value : json option; + } type dataTransformerConfig = { - id: string; - ?disabled: bool option; - ?filter: matcherConfig option; - options: json; - } + id : string; + ?disabled : bool option; + ?filter : matcherConfig option; + options : json; + } type dataSourceRef = { - ?type_ : string option; - ?uid: string option; - } + ?type_ : string option; + ?uid : string option; + } type dashboardLinkType = [ | Link | Dashboards - ] + ] type dashboardLink = { - title: string; - type_ : dashboardLinkType; - icon: string; - tooltip: string; - url: string; - tags: string list; - ~asDropdown : bool; - ~targetBlank : bool; - ~includeVars : bool; - ~keepTime : bool; - } - - type dashboardCursorSync = int + title : string; + type_ : dashboardLinkType; + icon : string; + tooltip : string; + url : string; + tags : string list; + ~asDropdown : bool; + ~targetBlank : bool; + ~includeVars : bool; + ~keepTime : bool; + } + + type dashboardCursorSync = int type annotationTarget = { - limit: int64; - matchAny: bool; - tags: string list; - type_ : string; - } + limit : int64; + matchAny : bool; + tags : string list; + type_ : string; + } type annotationQuery = { - name: string; - datasource: dataSourceRef; - ~enable : bool; - ~hide : bool; - iconColor: string; - ?filter: annotationPanelFilter option; - ?target: annotationTarget option; - ?type_ : string option; - ~builtIn : float; - } + name : string; + datasource : dataSourceRef; + ~enable : bool; + ~hide : bool; + iconColor : string; + ?filter : annotationPanelFilter option; + ?target : annotationTarget option; + ?type_ : string option; + ~builtIn : float; + } type annotationPanelFilter = { - ~exclude : bool; - ids: int list; - } + ~exclude : bool; + ids : int list; + } type annotationContainer = { - ?list: annotationQuery list option; - } + ?list : annotationQuery list option; + } diff --git a/tests/one_of.ml b/tests/one_of.ml index 506a12c..959e258 100644 --- a/tests/one_of.ml +++ b/tests/one_of.ml @@ -65,9 +65,10 @@ let nested_one_of_test _ = | String of string | Json of variableModelQueryJson ] + type variableModel = { - ?query: variableModelQuery option; + ?query : variableModelQuery option; }|} in assert_schema input output diff --git a/tests/smoke.t b/tests/smoke.t index 30d6344..00a6e90 100644 --- a/tests/smoke.t +++ b/tests/smoke.t @@ -5,8 +5,8 @@ Generate ATD out of JSON Scheme type int64 = int type product = { - ?productId: int option; - } + ?productId : int option; + } Generate ATD out of JSON Scheme with --format attribute $ jsonschema2atd --format jsonschema ./mocks/simple_jsonschema.json (* Generated by jsonschema2atd *) @@ -14,8 +14,8 @@ Generate ATD out of JSON Scheme with --format attribute type int64 = int type product = { - ?productId: int option; - } + ?productId : int option; + } Generate ATD out of JSON Scheme with -f attribute $ jsonschema2atd -f jsonschema ./mocks/simple_jsonschema.json (* Generated by jsonschema2atd *) @@ -23,8 +23,8 @@ Generate ATD out of JSON Scheme with -f attribute type int64 = int type product = { - ?productId: int option; - } + ?productId : int option; + } Generate ATD out of OpenAPI doc with --format attribute $ jsonschema2atd --format openapi ./mocks/simple_openapi.json (* Generated by jsonschema2atd *) @@ -32,8 +32,8 @@ Generate ATD out of OpenAPI doc with --format attribute type int64 = int type product = { - ?productId: int option; - } + ?productId : int option; + } Generate ATD out of OpenAPI doc with -f attribute $ jsonschema2atd -f openapi ./mocks/simple_openapi.json (* Generated by jsonschema2atd *) @@ -41,8 +41,8 @@ Generate ATD out of OpenAPI doc with -f attribute type int64 = int type product = { - ?productId: int option; - } + ?productId : int option; + } Generate ATD out of JSON Schema that contains defs $ jsonschema2atd --format=jsonschema ./mocks/jsonchema_defs.json @@ -50,12 +50,12 @@ Generate ATD out of JSON Schema that contains defs type json = abstract type int64 = int - type name = string + type name = string type root = { - first_name: name; - last_name: name; - } + first_name : name; + last_name : name; + } Generate ATD out of JSON Schema that contains definitions (legacy support) $ jsonschema2atd --format=jsonschema ./mocks/jsonchema_definitions.json @@ -63,9 +63,9 @@ Generate ATD out of JSON Schema that contains definitions (legacy support) type json = abstract type int64 = int - type name = string + type name = string type root = { - first_name: name; - last_name: name; - } + first_name : name; + last_name : name; + } From c8822f059b0f6057394d50b603b5253e64c09420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20Roch=C3=A9?= Date: Tue, 27 Feb 2024 16:11:22 +0100 Subject: [PATCH 4/6] jsonschema: support external references --- lib/generator.ml | 27 +++++++++++++------- tests/mocks/jsonschema_refs.json | 42 ++++++++++++++++++++++++++++++++ tests/smoke.t | 19 +++++++++++++++ 3 files changed, 79 insertions(+), 9 deletions(-) create mode 100644 tests/mocks/jsonschema_refs.json diff --git a/lib/generator.ml b/lib/generator.ml index ec081bc..edf5988 100644 --- a/lib/generator.ml +++ b/lib/generator.ml @@ -23,17 +23,26 @@ let process_int_type schema = | _ -> failwith "int has unextected format" let get_ref_name ref = - match String.split_on_char '/' ref with + let uri, pointer = + match String.split_on_char '#' ref with + | [ uri; pointer ] -> uri, Some pointer + | [ uri ] -> uri, None + | _ -> failwith (sprintf "Unsupported remote ref value: %s. The URI contains multiple '#'." ref) + in + let name_of_path path = + match path |> String.split_on_char '/' |> List.rev |> List.hd with + | exception _ -> failwith (sprintf "Unsupported ref value: %s" ref) + | name -> name + in + match pointer with + | None -> name_of_path uri + | Some pointer -> + match String.split_on_char '/' pointer with (* OpenAPI defs *) - | [ "#"; "components"; "schemas"; type_name ] -> type_name + | [ ""; "components"; "schemas"; type_name ] -> type_name (* JSON Schema defs *) - | [ "#"; ("$defs" | "definitions"); type_name ] -> type_name - | _ -> - failwith - (Printf.sprintf - "Unsupported ref value: %s. Supported ref URI are: #/components/schemas/* and #/$defs/* and #/definitions/*" - ref - ) + | [ ""; ("$defs" | "definitions"); type_name ] -> type_name + | _ -> name_of_path pointer let output = Buffer.create 16 let input_toplevel_schemas = ref [] diff --git a/tests/mocks/jsonschema_refs.json b/tests/mocks/jsonschema_refs.json new file mode 100644 index 0000000..de2045d --- /dev/null +++ b/tests/mocks/jsonschema_refs.json @@ -0,0 +1,42 @@ +{ + "$id": "https://example.com/schemas/customer", + "type": "object", + "properties": { + "first_name": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "shipping_address": { + "$ref": "/schemas/address" + }, + "billing_address": { + "$ref": "/schemas/address" + }, + "aa": { + "$ref": "https://example.com/schemas/address" + }, + "bb": { + "$ref": "http://example.com/schemas/address" + }, + "cc": { + "$ref": "http://example.com/schemas#/definitions/address" + }, + "dd": { + "$ref": "http://example.com/schemas#/definitions/address" + }, + "ee": { + "$ref": "#/definitions/schemas/address" + }, + "ff": { + "$ref": "#/definitions/address" + } + }, + "required": [ + "first_name", + "last_name", + "shipping_address", + "billing_address" + ] +} diff --git a/tests/smoke.t b/tests/smoke.t index 00a6e90..2b60b36 100644 --- a/tests/smoke.t +++ b/tests/smoke.t @@ -69,3 +69,22 @@ Generate ATD out of JSON Schema that contains definitions (legacy support) first_name : name; last_name : name; } + +Generate ATD out of JSON Schema that uses references + $ jsonschema2atd --format=jsonschema ./mocks/jsonschema_refs.json + (* Generated by jsonschema2atd *) + type json = abstract + type int64 = int + + type root = { + first_name : string; + last_name : string; + shipping_address : address; + billing_address : address; + ?aa : address option; + ?bb : address option; + ?cc : address option; + ?dd : address option; + ?ee : address option; + ?ff : address option; + } From 635d9937249f0460189a680921124430822dc567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20Roch=C3=A9?= Date: Tue, 27 Feb 2024 16:37:37 +0100 Subject: [PATCH 5/6] jsonschema: support non string enums --- lib/generator.ml | 25 +++++++++++++++++++-- lib/json_schema.atd | 2 +- tests/mocks/jsonschema_enums.json | 36 +++++++++++++++++++++++++++++++ tests/smoke.t | 24 +++++++++++++++++++++ 4 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 tests/mocks/jsonschema_enums.json diff --git a/lib/generator.ml b/lib/generator.ml index edf5988..559c086 100644 --- a/lib/generator.ml +++ b/lib/generator.ml @@ -141,7 +141,16 @@ let rec process_schema_type ~ancestors (schema : schema) = | Some schemas -> process_one_of ~ancestors schemas | None -> match schema.enum, schema.typ with - | Some enums, Some String -> process_enums enums + | Some enums, Some String -> process_string_enums enums + | Some _, Some Integer -> + (* this is more lenient than it should *) + maybe_nullable (process_int_type schema) + | Some _, Some Number -> + (* this is more lenient than it should *) + maybe_nullable "float" + | Some _, Some Boolean -> + (* this is more lenient than it should *) + maybe_nullable "bool" | Some _, _ -> failwith "only string enums are supported" | None, _ -> match schema.typ with @@ -217,7 +226,19 @@ and process_one_of ~ancestors (schemas_or_refs : schema or_ref list) = let variants = List.map make_one_of_variant schemas_or_refs |> String.concat "\n" in sprintf "[\n%s\n] " variants -and process_enums enums = +and process_string_enums enums = + let enums = + List.map + (function + | `String s -> s + | value -> + failwith + (sprintf "Invalid value %s in string enum %s" (Yojson.Basic.to_string value) + (Yojson.Basic.to_string (`List enums)) + ) + ) + enums + in let make_enum_variant value = sprintf {| | %s |} (variant_name value) value in let variants = List.map make_enum_variant enums |> String.concat "\n" in sprintf "[\n%s\n]" variants diff --git a/lib/json_schema.atd b/lib/json_schema.atd index 1ddd5bf..4df176f 100644 --- a/lib/json_schema.atd +++ b/lib/json_schema.atd @@ -60,7 +60,7 @@ type schema = { (* 6.1 validation for any instance type *) ~typ : typ nullable; - ~enum : string nonempty_list nullable; + ~enum : json nonempty_list nullable; (* 6.2 validation for numeric instances *) (* ~multiple_of : float nullable; *) diff --git a/tests/mocks/jsonschema_enums.json b/tests/mocks/jsonschema_enums.json new file mode 100644 index 0000000..830768d --- /dev/null +++ b/tests/mocks/jsonschema_enums.json @@ -0,0 +1,36 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "myProperty": { + "type": "string", + "enum": [ + "foo", + "bar" + ] + }, + "myPropertyInt": { + "type": "integer", + "enum": [ + 1, + 2 + ] + }, + "myPropertyNumber": { + "type": "number", + "enum": [ + 1.2, + 2.3 + ] + }, + "myPropertyBool": { + "type": "boolean", + "enum": [ + true + ] + } + }, + "required": [ + "myProperty" + ] +} diff --git a/tests/smoke.t b/tests/smoke.t index 2b60b36..772103a 100644 --- a/tests/smoke.t +++ b/tests/smoke.t @@ -88,3 +88,27 @@ Generate ATD out of JSON Schema that uses references ?ee : address option; ?ff : address option; } + +Generate ATD out of JSON Schema that uses enums + $ jsonschema2atd --format=jsonschema ./mocks/jsonschema_enums.json + (* Generated by jsonschema2atd *) + type json = abstract + type int64 = int + + type rootMyProperty = [ + | Foo + | Bar + ] + + type rootMyPropertyInt = int + + type rootMyPropertyNumber = float + + type rootMyPropertyBool = bool + + type root = { + myProperty : rootMyProperty; + ?myPropertyInt : rootMyPropertyInt option; + ?myPropertyNumber : rootMyPropertyNumber option; + ?myPropertyBool : rootMyPropertyBool option; + } From e3a68249514baf63c30534a40fa9a0408b57ccfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis=20Roch=C3=A9?= Date: Tue, 27 Feb 2024 18:05:42 +0100 Subject: [PATCH 6/6] cli: convert multiple schemas at the same time --- bin/main.ml | 31 +++++++++++++++++++++---------- lib/generator.ml | 13 +++++++------ tests/base.ml | 3 +-- tests/grok.t | 2 +- tests/smoke.t | 18 +++++++++--------- 5 files changed, 39 insertions(+), 28 deletions(-) diff --git a/bin/main.ml b/bin/main.ml index b73cf7c..66f1e12 100644 --- a/bin/main.ml +++ b/bin/main.ml @@ -12,18 +12,29 @@ module Input_format = struct let all = [ JSONSchema; OpenAPI ] end -let generate_atd input_format path_in = - let ic = open_in path_in in - let input_content = really_input_string ic (in_channel_length ic) in - close_in ic; - +let generate_atd input_format paths = let generate = match input_format with | Input_format.JSONSchema -> Generator.make_atd_of_jsonschema | OpenAPI -> Generator.make_atd_of_openapi in - input_content |> generate |> print_string; - () + print_endline (Generator.base (String.concat " " (List.map Filename.basename paths))); + let root = + match paths with + | [ _ ] -> `Default + | _ -> `Per_file + in + List.iter + (fun path -> + let root = + match root with + | `Default -> None + | `Per_file -> Some (path |> Filename.basename |> Filename.remove_extension |> Utils.sanitize_name) + in + let input_content = In_channel.with_open_bin path In_channel.input_all in + input_content |> generate ?root |> print_string + ) + paths let input_format_term = let formats = List.map (fun fmt -> Input_format.stringify fmt, fmt) Input_format.all in @@ -32,9 +43,9 @@ let input_format_term = Arg.(value & opt format JSONSchema & info [ "format"; "f" ] ~docv:"FORMAT" ~doc) let main = - let doc = "Generate an ATD file from a JSON Schema / OpenAPI document" in - let path_in = Arg.(required & pos 0 (some file) None & info [] ~docv:"input file" ~doc) in - let term = Term.(const generate_atd $ input_format_term $ path_in) in + let doc = "Generate an ATD file from a list of JSON Schema / OpenAPI document" in + let paths = Arg.(non_empty & pos_all file [] & info [] ~docv:"FILES" ~doc) in + let term = Term.(const generate_atd $ input_format_term $ paths) in let info = Cmd.info "jsonschema2atd" ~doc ~version:(Version.get ()) in Cmd.v info term diff --git a/lib/generator.ml b/lib/generator.ml index 559c086..6b338e4 100644 --- a/lib/generator.ml +++ b/lib/generator.ml @@ -255,11 +255,13 @@ let process_schemas (schemas : (string * schema or_ref) list) = ) [] schemas -let base = - {|(* Generated by jsonschema2atd *) +let base from = + sprintf + {|(* Generated by jsonschema2atd from %s *) type json = abstract type int64 = int |} + from let make_atd_of_schemas schemas = input_toplevel_schemas := @@ -270,20 +272,19 @@ let make_atd_of_schemas schemas = ) schemas; Buffer.clear output; - Buffer.add_string output (base ^ "\n"); Buffer.add_string output (String.concat "\n" (process_schemas schemas)); Buffer.contents output -let make_atd_of_jsonschema input = +let make_atd_of_jsonschema ?(root = "root") input = let schema = Json_schema_j.schema_of_string input in - let root_type_name = Option.value ~default:"root" schema.title in + let root_type_name = Option.value ~default:root schema.title in let defs = let defs = List.concat_map Utils.list_of_nonempty [ schema.defs; schema.definitions ] in List.map (fun (name, schema) -> name, Obj schema) defs in make_atd_of_schemas ([ root_type_name, Obj schema ] @ defs) -let make_atd_of_openapi input = +let make_atd_of_openapi ?root:_ input = let root = Openapi_j.root_of_string input in match root.components with | None -> failwith "components are empty" diff --git a/tests/base.ml b/tests/base.ml index 0e52734..539ec54 100644 --- a/tests/base.ml +++ b/tests/base.ml @@ -6,11 +6,10 @@ let openapi_json_template schemas = schemas let replace_whitespace str = Str.global_replace (Str.regexp "[ \t\n\r]+") "" str -let remove_prelude str = Str.global_replace (Str.regexp (Str.quote Generator.base)) "" str let test_strings_cmp a b = String.equal (replace_whitespace a) (replace_whitespace b) let assert_schema input output = assert_equal ~cmp:test_strings_cmp ~printer:(fun str -> str) output - (remove_prelude (Generator.make_atd_of_openapi (openapi_json_template input))) + (Generator.make_atd_of_openapi (openapi_json_template input)) diff --git a/tests/grok.t b/tests/grok.t index 951de8e..d0511d6 100644 --- a/tests/grok.t +++ b/tests/grok.t @@ -1,6 +1,6 @@ Generate ATD types from grok (Grafana Object Development Kit) dashboard types $ jsonschema2atd --format openapi ./mocks/dashboard_types_gen.json - (* Generated by jsonschema2atd *) + (* Generated by jsonschema2atd from dashboard_types_gen.json *) type json = abstract type int64 = int diff --git a/tests/smoke.t b/tests/smoke.t index 772103a..a75ba66 100644 --- a/tests/smoke.t +++ b/tests/smoke.t @@ -1,6 +1,6 @@ Generate ATD out of JSON Scheme $ jsonschema2atd ./mocks/simple_jsonschema.json - (* Generated by jsonschema2atd *) + (* Generated by jsonschema2atd from simple_jsonschema.json *) type json = abstract type int64 = int @@ -9,7 +9,7 @@ Generate ATD out of JSON Scheme } Generate ATD out of JSON Scheme with --format attribute $ jsonschema2atd --format jsonschema ./mocks/simple_jsonschema.json - (* Generated by jsonschema2atd *) + (* Generated by jsonschema2atd from simple_jsonschema.json *) type json = abstract type int64 = int @@ -18,7 +18,7 @@ Generate ATD out of JSON Scheme with --format attribute } Generate ATD out of JSON Scheme with -f attribute $ jsonschema2atd -f jsonschema ./mocks/simple_jsonschema.json - (* Generated by jsonschema2atd *) + (* Generated by jsonschema2atd from simple_jsonschema.json *) type json = abstract type int64 = int @@ -27,7 +27,7 @@ Generate ATD out of JSON Scheme with -f attribute } Generate ATD out of OpenAPI doc with --format attribute $ jsonschema2atd --format openapi ./mocks/simple_openapi.json - (* Generated by jsonschema2atd *) + (* Generated by jsonschema2atd from simple_openapi.json *) type json = abstract type int64 = int @@ -36,7 +36,7 @@ Generate ATD out of OpenAPI doc with --format attribute } Generate ATD out of OpenAPI doc with -f attribute $ jsonschema2atd -f openapi ./mocks/simple_openapi.json - (* Generated by jsonschema2atd *) + (* Generated by jsonschema2atd from simple_openapi.json *) type json = abstract type int64 = int @@ -46,7 +46,7 @@ Generate ATD out of OpenAPI doc with -f attribute Generate ATD out of JSON Schema that contains defs $ jsonschema2atd --format=jsonschema ./mocks/jsonchema_defs.json - (* Generated by jsonschema2atd *) + (* Generated by jsonschema2atd from jsonchema_defs.json *) type json = abstract type int64 = int @@ -59,7 +59,7 @@ Generate ATD out of JSON Schema that contains defs Generate ATD out of JSON Schema that contains definitions (legacy support) $ jsonschema2atd --format=jsonschema ./mocks/jsonchema_definitions.json - (* Generated by jsonschema2atd *) + (* Generated by jsonschema2atd from jsonchema_definitions.json *) type json = abstract type int64 = int @@ -72,7 +72,7 @@ Generate ATD out of JSON Schema that contains definitions (legacy support) Generate ATD out of JSON Schema that uses references $ jsonschema2atd --format=jsonschema ./mocks/jsonschema_refs.json - (* Generated by jsonschema2atd *) + (* Generated by jsonschema2atd from jsonschema_refs.json *) type json = abstract type int64 = int @@ -91,7 +91,7 @@ Generate ATD out of JSON Schema that uses references Generate ATD out of JSON Schema that uses enums $ jsonschema2atd --format=jsonschema ./mocks/jsonschema_enums.json - (* Generated by jsonschema2atd *) + (* Generated by jsonschema2atd from jsonschema_enums.json *) type json = abstract type int64 = int