From 39b21b23ece9fe8171b50dcd5a99df46c85b7f78 Mon Sep 17 00:00:00 2001 From: Oliver Schmid Date: Mon, 27 Nov 2023 11:44:32 +0100 Subject: [PATCH] initial commit --- .github/workflows/build.yml | 49 ++++++ .gitignore | 4 + build.py | 70 +++++++++ generator/__init__.py | 0 generator/templates/OpenMINDS.java.j2 | 88 +++++++++++ generator/templates/interface.java.j2 | 18 +++ generator/templates/schema_class.java.j2 | 78 ++++++++++ pipeline/__init__.py | 0 pipeline/translator.py | 190 +++++++++++++++++++++++ pipeline/utils.py | 23 +++ requirements.txt | 2 + 11 files changed, 522 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100644 .gitignore create mode 100644 build.py create mode 100644 generator/__init__.py create mode 100644 generator/templates/OpenMINDS.java.j2 create mode 100644 generator/templates/interface.java.j2 create mode 100644 generator/templates/schema_class.java.j2 create mode 100644 pipeline/__init__.py create mode 100644 pipeline/translator.py create mode 100644 pipeline/utils.py create mode 100644 requirements.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..fc542d0 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,49 @@ +# MIT licensed +name: openMINDS_java_build_pipeline + +on: + push: + branches: + - pipeline + workflow_dispatch: # This triggers the workflow when a webhook is received + + +jobs: + build: + runs-on: ubuntu-latest + steps: + + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Set up Python 3.11 + uses: actions/setup-python@v2 + with: + python-version: 3.11 + + - name: Run build + run: | + pip install -r requirements.txt + python build.py + + - name: Checkout main branch + uses: actions/checkout@v3 + with: + ref: main + path: main + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Push to main + run: | + cp -R target/* main + cd main + rm -rf build openMINDS.egg-info + git config --global user.email "openminds@ebrains.eu" + git config --global user.name "openMINDS pipeline" + if [[ $(git add . --dry-run | wc -l) -gt 0 ]]; then + git add . + git commit -m "build triggered by submodule ${{ inputs.repository }} version ${{ inputs.branch }}" + git push -f + else + echo "Nothing to commit" + fi \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..23109ba --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +sources/** +target/** +.idea/** +src/** \ No newline at end of file diff --git a/build.py b/build.py new file mode 100644 index 0000000..22c24a9 --- /dev/null +++ b/build.py @@ -0,0 +1,70 @@ +import os.path +import shutil + +from jinja2 import Environment, PackageLoader, select_autoescape + +from pipeline.translator import JavaBuilder +from pipeline.utils import clone_sources, SchemaLoader + +print("***************************************") +print(f"Triggering the generation of Java classes for openMINDS") +print("***************************************") + +# Step 1 - clone central repository in main branch to get the latest sources +clone_sources() +schema_loader = SchemaLoader() +if os.path.exists("target"): + shutil.rmtree("target") + +ignored_versions = ["v1.0", "v2.0"] +relevant_versions = [v for v in schema_loader.get_schema_versions() if v not in ignored_versions] + + +def build_central(): + env = Environment( + loader=PackageLoader("generator"), + autoescape=select_autoescape() + ) + template = env.get_template("OpenMINDS.java.j2") + result = template.render( + relevant_versions = built_versions, + packages_by_version = packages_by_version + ) + target_file = "target/src/main/java/org/openmetadatainitiative/openminds/OpenMINDS.java" + os.makedirs(os.path.dirname(target_file), exist_ok=True) + with open(target_file, "w") as target_file: + target_file.write(result) + +built_versions = [] +packages_by_version = {} + + +for schema_version in relevant_versions: + # Step 2 - find all involved schemas for the current version + schemas_file_paths = schema_loader.find_schemas(schema_version) + type_to_class_name = {} + implemented_interfaces = {} + builders = [] + for schema_file_path in schemas_file_paths: + builder = JavaBuilder(schema_file_path, schema_loader.schemas_sources) + if builder.version not in packages_by_version: + packages_by_version[builder.version] = {} + if builder.relative_path_without_extension[0] not in packages_by_version[builder.version]: + packages_by_version[builder.version][builder.relative_path_without_extension[0]] = [] + packages_by_version[builder.version][builder.relative_path_without_extension[0]].append((builder.class_name, builder.canonical_class_name())) + if builder.version not in built_versions: + built_versions.append(builder.version) + for property, types in builder.additional_interfaces.items(): + for t in types: + if t not in implemented_interfaces: + implemented_interfaces[t] = [] + implemented_interfaces[t].append(f"{builder.interface_path(property).replace('/', '.')}") + type_to_class_name[builder.type]=builder.canonical_class_name() + builders.append(builder) + + + for builder in builders: + # Step 3 - translate and build each openMINDS schema as JSON-Schema + builder.build(type_to_class_name, implemented_interfaces) + +build_central() \ No newline at end of file diff --git a/generator/__init__.py b/generator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/generator/templates/OpenMINDS.java.j2 b/generator/templates/OpenMINDS.java.j2 new file mode 100644 index 0000000..d095cda --- /dev/null +++ b/generator/templates/OpenMINDS.java.j2 @@ -0,0 +1,88 @@ +package org.openmetadatainitiative.openminds; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.openmetadatainitiative.openminds.utils.Instance; +import org.openmetadatainitiative.openminds.utils.OpenMINDSContext; +import org.openmetadatainitiative.openminds.utils.ParsingUtils; +import org.openmetadatainitiative.openminds.utils.LocalId; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; + +/** + * ATTENTION! This is an autogenerated file based on the openMINDS schema - do not apply manual changes since they are going to be overwritten. + */ +public class OpenMINDS { + + private final OpenMINDSContext context; + + private OpenMINDS(OpenMINDSContext context){ + this.context = context; + } + + private static void persist(String targetDirectory, Stream instances){ + File dir = new File(targetDirectory); + if(!dir.exists()){ + dir.mkdirs(); + } + instances.forEach(i -> { + File f = new File(targetDirectory+File.separator+i.getLocalId().id()+".jsonld"); + try { + ParsingUtils.OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValue(f, i); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + {% for version in relevant_versions %} + public static OpenMINDS.{{version[0]|upper}}{{version[1:]}} {{version}}(){ + return {{version}}(OpenMINDSContext.defaultContext()); + } + + public static OpenMINDS.{{version[0]|upper}}{{version[1:]}} {{version}}(String idPrefix){ + return {{version}}(new OpenMINDSContext(idPrefix, true)); + } + + private static OpenMINDS.{{version[0]|upper}}{{version[1:]}} {{version}}(OpenMINDSContext context) { + return new OpenMINDS(context).new {{version[0]|upper}}{{version[1:]}}(); + } + + + + public class {{version[0]|upper}}{{version[1:]}} { + + private final List> builders = new ArrayList<>(); + + {% for package in packages_by_version[version].keys() %} + public final OpenMINDS.{{version[0]|upper}}{{version[1:]}}.{{package[0]|upper}}{{package[1:]}} {{package}} = new {{package[0]|upper}}{{package[1:]}}(); + + + public class {{package[0]|upper}}{{package[1:]}}{ + + {% for class in packages_by_version[version][package] %} + public final OpenMINDS.{{version[0]|upper}}{{version[1:]}}.{{package[0]|upper}}{{package[1:]}}.{{class[0]}} {{class[0][0]|lower}}{{class[0][1:]}} = new {{class[0]}}(); + + public class {{class[0]}} { + + public {{class[1]}}.Builder create(String localId){ + final {{class[1]}}.Builder builder = {{class[1]}}.create(new LocalId(localId)); + builders.add(builder); + return builder; + } + } + {% endfor %} + } + {% endfor %} + + public void persist(String targetDirectory) { + OpenMINDS.persist(targetDirectory, builders.stream().map(Builder::build)); + } + } + + {% endfor %} + +} diff --git a/generator/templates/interface.java.j2 b/generator/templates/interface.java.j2 new file mode 100644 index 0000000..cd141ee --- /dev/null +++ b/generator/templates/interface.java.j2 @@ -0,0 +1,18 @@ +package {{package_name}}; + +import org.openmetadatainitiative.openminds.utils.ByTypeDeserializer; +import org.openmetadatainitiative.openminds.utils.Entity; +import org.openmetadatainitiative.openminds.utils.Reference; + +/** + * ATTENTION! This is an autogenerated file based on the openMINDS schema - do not apply manual changes since they are going to be overwritten. + */ +public interface {{additional_interface}} extends Entity { + Reference getReference(); + + class Deserializer extends ByTypeDeserializer<{{additional_interface}}> { + public Deserializer() { + super({{types|join(', ')}}); + } + } +} \ No newline at end of file diff --git a/generator/templates/schema_class.java.j2 b/generator/templates/schema_class.java.j2 new file mode 100644 index 0000000..3050ae5 --- /dev/null +++ b/generator/templates/schema_class.java.j2 @@ -0,0 +1,78 @@ +package {{package_name}}; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.openmetadatainitiative.openminds.utils.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +{% for import in imports %}import {{import}}; +{% endfor %} + +import static {{package_name}}.{{ class_name }}.*; +/** + {{java_doc}} + * + * ATTENTION! This is an autogenerated file based on the openMINDS schema - do not apply manual changes since they are going to be overwritten. + */ +@InstanceType(SEMANTIC_NAME) +@JsonIgnoreProperties(ignoreUnknown = true) +public class {{ class_name }} extends Instance {% if implemented_interfaces %}implements {{ implemented_interfaces|join(', ') }}{% endif %}{ + static final String SEMANTIC_NAME = "{{ type }}"; + + @JsonIgnore + public Reference<{{ class_name }}> getReference() { + return doGetReference(); + } + + public static Reference<{{ class_name }}> createReference(InstanceId instanceId) { + return new Reference<>(instanceId); + } + + private {{ class_name }}(LocalId localId ) { + super(localId); + } + + + public class Builder implements org.openmetadatainitiative.openminds.utils.Builder<{{ class_name }}>{ + {% for property in properties %}{% for line in builder_for_properties[property] %} + {{ line }} + {% endfor %}{% endfor %} + + public {{ class_name }} build() { + if ({{ class_name }}.this.id == null) { + {{ class_name }}.this.id = new InstanceId(UUID.randomUUID().toString()); + } + if({{ class_name }}.this.types == null || {{ class_name }}.this.types.isEmpty() || !{{ class_name }}.this.types.contains(SEMANTIC_NAME)){ + final List oldValues = {{ class_name }}.this.types; + {{ class_name }}.this.types = new ArrayList<>(); + {{ class_name }}.this.types.add(SEMANTIC_NAME); + if(oldValues != null){ + {{ class_name }}.this.types.addAll(oldValues); + } + } + return {{ class_name }}.this; + } + } + +{% for property in properties %} @JsonProperty(value = "{%if property in absolute_property_translations%}{{absolute_property_translations[property]}}{%else%}{{property}}{%endif%}") + {{member_for_properties[property]}} + {%if property in property_descriptions and property_descriptions[property] %} + /** + * {{property_descriptions[property]}} + */{% endif %} + {{getter_for_properties[property]}} + + {% endfor %} + public static {{ class_name }}.Builder create(LocalId localId){ + return new {{ class_name }}(localId).new Builder(); + } + + public {{ class_name }}.Builder copy(){ + return ParsingUtils.OBJECT_MAPPER.convertValue(this, {{ class_name }}.class).new Builder(); + } +} diff --git a/pipeline/__init__.py b/pipeline/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pipeline/translator.py b/pipeline/translator.py new file mode 100644 index 0000000..988e69a --- /dev/null +++ b/pipeline/translator.py @@ -0,0 +1,190 @@ +import json +import os.path +from typing import Dict, List +from jinja2 import Environment, PackageLoader, select_autoescape + +#Some property names can conflict with protected keywords of the target language. Accordingly, they need to be translated +property_translation = { + "abstract": "abstract_" +} + + +class JavaBuilder(object): + + def __init__(self, schema_file_path:str, root_path:str): + _relative_path_without_extension = schema_file_path[len(root_path)+1:].replace(".schema.omi.json", "").split("/") + self.version = _relative_path_without_extension[0].split(".")[0] + self.relative_path_without_extension = _relative_path_without_extension[1:] + self.absolute_translations = {} + with open(schema_file_path, "r") as schema_f: + self._schema_payload = json.load(schema_f) + if "properties" in self._schema_payload: + for p in set(self._schema_payload["properties"].keys()): + property_split = p.split('/') + simple_property_name = property_split[-1] + path = "/".join(property_split[0:-1]) + if simple_property_name in property_translation: + self.absolute_translations[f"{path}/{property_translation[simple_property_name]}"] = p + self._schema_payload["properties"][f"{path}/{property_translation[simple_property_name]}"] = self._schema_payload["properties"][p] + del self._schema_payload["properties"][p] + if "required" in self._schema_payload: + for p in self._schema_payload["required"]: + property_split = p.split('/') + simple_property_name = property_split[-1] + path = "/".join(property_split[0:-1]) + if simple_property_name in property_translation: + self.absolute_translations[f"{path}/{property_translation[simple_property_name]}"] = p + self._schema_payload["required"].remove(p) + self._schema_payload["required"].append(f"{path}/{property_translation[simple_property_name]}") + + self.type = self._schema_payload['_type'] + self.version_path = os.path.join("org/openmetadatainitiative/openminds/", self.version) + self.package_path = os.path.join(self.version_path, "/".join(self.relative_path_without_extension[0:-1])).replace("-", "") + self.class_name = f"{self.relative_path_without_extension[-1][0].upper()}{self.relative_path_without_extension[-1][1:]}" + self.additional_interfaces = self._additional_interfaces() + + + + env = Environment( + loader=PackageLoader("generator"), + autoescape=select_autoescape() + ) + self.template = env.get_template("schema_class.java.j2") + self.interface_template = env.get_template("interface.java.j2") + self.imports = [] + + def property_to_type(self, property_name): + simple_property_name = property_name.split('/')[-1] + return f"{simple_property_name[0].upper()}{simple_property_name[1:]}" + + def _get_type(self, property_name, property_definition, type_to_class_name) -> str: + if "type" in property_definition: + if property_definition["type"] == "array": + if "_linkedTypes" in property_definition and property_definition["_linkedTypes"]: + if len(property_definition["_linkedTypes"]) == 1: + if property_definition["_linkedTypes"][0] in type_to_class_name: + self.imports.append(type_to_class_name[property_definition["_linkedTypes"][0]]) + return f"List>" + else: + self.imports.append(self.interface_path(property_name).replace("/", ".")) + return f"List>" + if "_embeddedTypes" in property_definition and property_definition["_embeddedTypes"]: + if len(property_definition["_embeddedTypes"]) == 1: + if property_definition["_embeddedTypes"][0] in type_to_class_name: + self.imports.append(type_to_class_name[property_definition["_embeddedTypes"][0]]) + return f"List<{property_definition['_embeddedTypes'][0].split('/')[-1]}>" + else: + self.imports.append(self.interface_path(property_name).replace("/", ".")) + return f"List" + if "items" in property_definition: + if "type" in property_definition["items"]: + if property_definition["items"]["type"] == "string": + return "List" + if property_definition["type"] == "string": + return "String" + else: + if "_linkedTypes" in property_definition and property_definition["_linkedTypes"]: + if len(property_definition["_linkedTypes"]) == 1: + if property_definition["_linkedTypes"][0] in type_to_class_name: + self.imports.append(type_to_class_name[property_definition["_linkedTypes"][0]]) + return f"Reference<{property_definition['_linkedTypes'][0].split('/')[-1]}>" + else: + self.imports.append(self.interface_path(property_name).replace("/", ".")) + return f"Reference" + if "_embeddedTypes" in property_definition and property_definition["_embeddedTypes"]: + if len(property_definition["_embeddedTypes"]) == 1: + if property_definition["_embeddedTypes"][0] in type_to_class_name: + self.imports.append(type_to_class_name[property_definition["_embeddedTypes"][0]]) + return f"{property_definition['_embeddedTypes'][0].split('/')[-1]}" + else: + self.imports.append(self.interface_path(property_name).replace("/", ".")) + return f"{self.interface_name(property_name)}" + + return "Object" + + def translate_interface(self, additional_interface, types, package_name): + return self.interface_template.render( + package_name = package_name, + additional_interface = additional_interface, + types = types + ) + + + def translate(self, type_to_class_name, implemented_interfaces) -> str: + # set required properties + required_prop = self._schema_payload["required"] if "required" in self._schema_payload else [] + required_prop.extend(["@id", "@type"]) + + # set description + description = "* "+"\n * ".join(self._schema_payload['description'].split("\n")) if "description" in self._schema_payload else "* " + + property_descriptions = {} + member_for_properties = {} + getter_for_properties = {} + builder_for_properties = {} + + for property, spec in self._schema_payload["properties"].items(): + property_descriptions[property] = spec["description"] if "description" in spec else None + type = self._get_type(property, spec, type_to_class_name) + property_name = property.split('/')[-1] + member_for_properties[property] = f"private {type} {property_name};" + getter_for_properties[property] = f"public {type} get{property_name[0].upper()}{property_name[1:]}() {{\n return this.{property_name};\n }}" + builder_for_properties[property] = [ + f"public Builder {property_name}({type} {property_name}) {{ {self.class_name}.this.{property_name} = {property_name}; return this; }}" + ] + + if self.canonical_class_name() in self.imports: + self.imports.remove(self.canonical_class_name()) + + return self.template.render( + type = self.type, + java_doc = description, + package_name = self.package_path.replace("/", "."), + class_name = self.class_name, + properties = sorted(list(self._schema_payload["properties"].keys())), + member_for_properties = member_for_properties, + getter_for_properties = getter_for_properties, + property_descriptions = property_descriptions, + absolute_property_translations = self.absolute_translations, + implemented_interfaces = implemented_interfaces[self.type] if self.type in implemented_interfaces else None, + additional_interfaces = {self.property_to_type(p) : sorted([f"{t.split('/')[-1]}.class" for t in k]) for p, k in self.additional_interfaces.items()}, + builder_for_properties = builder_for_properties, + imports = sorted(list(set(self.imports))) + ) + + def _target_file_path(self) -> str: + return os.path.join(self.package_path, self.class_name) + + def canonical_class_name(self) -> str: + return self._target_file_path().replace("/", ".") + + def interface_name(self, property_name) -> str: + return f"{self.class_name}{self.property_to_type(property_name)}" + + def interface_path(self, property_name) -> str: + return os.path.join(self.package_path, "intf", self.interface_name(property_name)) + + def _additional_interfaces(self) -> Dict[str, List[str]]: + additional_interfaces = {} + for property, property_definition in self._schema_payload["properties"].items(): + if "_linkedTypes" in property_definition and len(property_definition["_linkedTypes"]) > 1: + additional_interfaces[property] = property_definition["_linkedTypes"] + elif "_embeddedTypes" in property_definition and len(property_definition["_embeddedTypes"]) > 1: + additional_interfaces[property] = property_definition["_embeddedTypes"] + return additional_interfaces + + def build(self, type_to_class_name, implemented_interfaces): + target_file = os.path.join("target", "src/main/java", f"{self._target_file_path()}.java") + os.makedirs(os.path.dirname(target_file), exist_ok=True) + result = self.translate(type_to_class_name, implemented_interfaces) + + with open(target_file, "w") as target_file: + target_file.write(result) + + for additional_interface, types in self.additional_interfaces.items(): + interface_path = self.interface_path(additional_interface) + target_file = os.path.join("target", "src/main/java", f"{interface_path}.java") + os.makedirs(os.path.dirname(target_file), exist_ok=True) + result = self.translate_interface(self.interface_name(additional_interface), [f"{type_to_class_name[t]}.class" for t in types if t in type_to_class_name], os.path.join(self.package_path, "intf").replace("/", ".")) + with open(target_file, "w") as target_file: + target_file.write(result) \ No newline at end of file diff --git a/pipeline/utils.py b/pipeline/utils.py new file mode 100644 index 0000000..42ca6f8 --- /dev/null +++ b/pipeline/utils.py @@ -0,0 +1,23 @@ +import glob +import os +import shutil +from typing import List + +from git import Repo, GitCommandError + +def clone_sources(): + if os.path.exists("sources"): + shutil.rmtree("sources") + Repo.clone_from("https://github.com/openMetadataInitiative/openMINDS.git", to_path="sources", depth=1) + +class SchemaLoader(object): + + def __init__(self): + self._root_directory = os.path.realpath(".") + self.schemas_sources = os.path.join(self._root_directory, "sources", "schemas") + + def get_schema_versions(self) -> List[str]: + return os.listdir(self.schemas_sources) + + def find_schemas(self, version:str) -> List[str]: + return glob.glob(os.path.join(self.schemas_sources, version, f'**/*.schema.omi.json'), recursive=True) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9e25c84 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +gitpython +jinja2 \ No newline at end of file