Merge pull request #219 from ansforge/auto/model_tracker

[AUTO] Model Tracker

.github/workflows/generate-model.yaml

@@ -1,9 +1,9 @@
 name: Generate Model
 
 on:
-  # pull_request:
-  #   branches:
-  #     - '**'
+  pull_request:
+    branches:
+      - '**'
   release:
     types:
       - published
@@ -53,13 +53,20 @@ jobs:
 
       - name: Clean up old generated schemas
         working-directory: ./src/main/resources
+        if: steps.filter.outputs.parsing_required == 'true'
         run: find ./json-schema -type f -name '*.json' ! -name 'customContent.schema.json' ! -name 'EDXL-DE-*.schema.json' -exec rm {} +
 
       - name: Clean up old generated java classes
         working-directory: ./src/main/java/com/hubsante/model
+        if: steps.filter.outputs.parsing_required == 'true'
         # We specifically only remove FOLDERS with the exception of a couple manually created ones
         run: find . -mindepth 1 -maxdepth 1 -type d ! -name 'builders' ! -name 'config' ! -name 'custom' ! -name 'edxl' ! -name 'exception' ! -name 'report' -exec rm -r {} +
 
+      - name: Run csv_parser to generate schemas.yaml
+        working-directory: ./csv_parser
+        if: steps.filter.outputs.parsing_required == 'true'
+        run: python workflow.py --stage output_schemas_yaml
+
       - name: Run csv_parser and collect OpenAPI & JSON Schemas
         working-directory: ./csv_parser
         if: steps.filter.outputs.parsing_required == 'true'
@@ -72,11 +79,6 @@ jobs:
           rm -r ./out/test-cases || true
           python workflow.py --stage test_case_parser
 
-      - name: Run csv_parser to generate schemas.yaml
-        working-directory: ./csv_parser
-        if: steps.filter.outputs.parsing_required == 'true'
-        run: python workflow.py --stage output_schemas_yaml
-
       - name: Collect schemas.yaml and copy it to json_schema2xsd
         working-directory: ./csv_parser
         if: steps.filter.outputs.parsing_required == 'true'
@@ -142,28 +144,35 @@ jobs:
 
       - name: Install node env 🏗
         uses: actions/setup-node@v4
+        if: steps.filter.outputs.parsing_required == 'true'
         with:
           node-version: 16
 
       - name: Install openapi-generator-cli
-        run: npm install -g @openapitools/openapi-generator-cli
+        if: steps.filter.outputs.parsing_required == 'true'
+        run: npm install -g @openapitools/[email protected]
 
       - name: Install linter
+        if: steps.filter.outputs.parsing_required == 'true'
         run: sudo apt install -y clang-format
 
       - name: Generate Java classes
+        if: steps.filter.outputs.parsing_required == 'true'
         working-directory: ./generator
         run: |
+          # Setup OpenAPI version
+          npx openapi-generator-cli version-manager set 7.4.0
           # Iterate over each file in the ./config directory, including the entire subfolder structure
           # and then run @openapitools/openapi-generator-cli generate for each file found
           # Important notice:          
           # Results of the find command are sorted in an alphabetic order before being passed to xargs
           # This means that since the order of class generation is important, it's necessary to maintain an adequately 
           # named file structure in the ./config/** directories
           # generator-config.json (if exists) -> usecase.generator-config.json -> wrapper.generator-config.json
-          find ./config/ -type f | sort -n | while read -r file; do npx @openapitools/openapi-generator-cli generate -c "$file" --skip-validate-spec; done
-          
+          find ./config/ -type f | sort -n | while read -r file; do npx @openapitools/openapi-generator-cli@2.9.0 generate -c "$file" --skip-validate-spec; done
+
       - name: Replace src/ with generated classes
+        if: steps.filter.outputs.parsing_required == 'true'
         run: |
           rm -r ./src/main/java/com/hubsante/model/rcde || true
           rm -r ./src/main/java/com/hubsante/model/cisu || true
@@ -180,27 +189,31 @@ jobs:
         run: chmod +x ./gradlew
 
       - name: Apply license
+        if: steps.filter.outputs.parsing_required == 'true'
         run: ./gradlew licenseFormat
 
       - name: Delete old xml files
+        if: steps.filter.outputs.parsing_required == 'true'
         run: |
           find ./src/main/resources/sample/examples -name "*.xml" -type f -delete
 
       - name: Generate XML files
+        if: steps.filter.outputs.parsing_required == 'true'
         run: |
           ./gradlew generateXml
         continue-on-error: true
 
       - name: Commit and push changes
         uses: stefanzweifel/git-auto-commit-action@v5
+        if: steps.filter.outputs.parsing_required == 'true'
         with:
           commit_message: ⚙️ Auto-génération des classes et des specs
 
       - name: Build and run TECHNICAL tests
         run: ./gradlew test --info -Ptechnical=true
 
       - name: Build and run REAL MESSAGES tests
-        run: ./gradlew test --info
+        run: ./gradlew test --info -Pfunctional=true
 
       # Ref.: https://github.com/gradle/gradle-build-action/issues/619#issuecomment-1453244157
       - name: Upload test report

.gitignore

@@ -9,7 +9,8 @@ coveragereport/**
 web_specs/generated/**
 bin/
 
-**/openapitools.json
+# À versionner pour suivre la version d'OpenAPI utilisé pour la génération
+# **/openapitools.json
 
 # Generated libraries packages
 # Python

automatic-schema-generator/templates/ContentMessage.java.tmpl

@@ -9,18 +9,18 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
 
 {{ range (datasource "config").schemas }}
 {{- if eq .automaticGeneration "Y" -}}
-import com.hubsante.model.{{ .package }}.{{ .rootElement | title }};
-import com.hubsante.model.{{ .package }}.{{ .rootElement | title }}Wrapper;
-{{ end }}{{ end -}}
+import com.hubsante.model.{{ .package }}.{{ .rootElement | title }};{{ if ne .rootElement "distributionElement" }}
+import com.hubsante.model.{{ .package }}.{{ .rootElement | title }}Wrapper;{{ end }}{{ end }}
+{{ end -}}
 import com.hubsante.model.report.ErrorWrapper;
 import com.hubsante.model.custom.CustomMessage;
 import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 @JsonTypeInfo(use = JsonTypeInfo.Id.DEDUCTION)
-@JsonSubTypes({ {{ range $i, $e := (datasource "config").schemas }}{{if eq .automaticGeneration "Y"}}
-        @JsonSubTypes.Type({{ .rootElement | title }}Wrapper.class),{{end}}{{end}}
+@JsonSubTypes({ {{ range $i, $e := (datasource "config").schemas }}{{if eq .automaticGeneration "Y"}}{{ if ne .rootElement "distributionElement" }}
+        @JsonSubTypes.Type({{ .rootElement | title }}Wrapper.class),{{ end }}{{end}}{{end}}
         @JsonSubTypes.Type(ErrorWrapper.class),
         @JsonSubTypes.Type(CustomMessage.class)
 })

automatic-schema-generator/templates/RC-XML-ContentType.xsd.tmpl

@@ -4,7 +4,7 @@
     This file has been generated by the automatic schema generator script.
 ********************************************************************************************************************
 -->
-<schema xmlns:x="urn:emergency:{{ $namespace := "cisu:3.0" }}{{ $namespace }}"
+<schema xmlns:x="urn:emergency:{{ $namespace := "eda:1.9" }}{{ $namespace }}"
         xmlns:xs="http://www.w3.org/2001/XMLSchema" 
         {{ range (datasource "config").schemas }}{{if and (eq .automaticGeneration "Y") .xmlns}}xmlns:{{.rootElement}}="urn:emergency:{{.xmlns}}"{{end}}
         {{end -}}

automatic-schema-generator/templates/schema.usecase.generator-config.json.tmpl

@@ -17,7 +17,7 @@
     "serializationLibrary": "jackson",
     "openApiNullable": true,
     "supportUrlQuery": false,
-    "xmlns": "{{if .xmlns}}{{.xmlns}}{{end}}{{if not .xmlns}}cisu:3.0:{{.rootElement}}{{end}}",
+    "xmlns": "{{if .xmlns}}{{.xmlns}}{{end}}{{if not .xmlns}}eda:1.9:{{.rootElement}}{{end}}",
     "enablePostProcessFile": true
   }
 }

automatic-schema-generator/templates/schema.wrapper.generator-config.json.tmpl

@@ -17,7 +17,7 @@
     "serializationLibrary": "jackson",
     "openApiNullable": true,
     "supportUrlQuery": false,
-    "xmlns": "cisu:3.0",
+    "xmlns": "eda:1.9",
     "enablePostProcessFile": true{{ if .customExtendPackage}},
     "customExtend": {
       "package": "{{ .customExtendPackage }}",

build.gradle

@@ -47,13 +47,6 @@ jacocoTestReport {
         xml.required = true
     }
 }
-
-test {
-    if (project.hasProperty('technical')) {
-        include '**/*Technical*.class'
-    } else if (!project.hasProperty('allTests'))
-        exclude '**/*Technical*.class'
-}
 
 allprojects {
     tasks.withType(JavaCompile) {
@@ -63,6 +56,11 @@ allprojects {
 
 tasks.named('test') {
     useJUnitPlatform()
+    if (project.hasProperty('functional')) {
+        exclude '**/*Technical*.class'
+    } else if (project.hasProperty('technical')) {
+        include '**/*Technical*.class'
+    }
     finalizedBy jacocoTestReport
 }
 

csv_parser/auto.sh

@@ -3,13 +3,15 @@
 # Set variables
 NOMENCLATURE_IN_FOLDER="/Users/romainfouilland/Library/CloudStorage/OneDrive-SharedLibraries-ANS/Espace Projets - Espace Programme SI-SAMU/01 - Equipe projet/07 - Innovation et prospectif/12 - Hub Santé/17 - MDD/Nomenclatures/01 - Base interne/"
 NOMENCLATURE_FOLDER="../nomenclature_parser/in/"
+TEST_CASE_SOURCES_FOLDER="./test-case-sources/"
 MODELS_IN_FILE="/Users/romainfouilland/Library/CloudStorage/OneDrive-SharedLibraries-ANS/Espace Projets - Espace Programme SI-SAMU/01 - Equipe projet/07 - Innovation et prospectif/12 - Hub Santé/17 - MDD/MDD - Hub Santé.xlsx"
 TECHNICAL_MODELS_IN_FILE="/Users/romainfouilland/Library/CloudStorage/OneDrive-SharedLibraries-ANS/Espace Projets - Espace Programme SI-SAMU/01 - Equipe projet/07 - Innovation et prospectif/12 - Hub Santé/17 - MDD/MDD TECHNICAL - Hub Santé.xlsx"
 MODELS_FILE="models/model.xlsx"
 TECHNICAL_MODELS_FILE="models/model-technical.xlsx"
 TRACKING_BRANCH_NAME="auto/model_tracker"
 DATE=$(date +'%y.%m.%d %H:%M')
 LOG_FILE="cron.log"
+PYTHON_CMD="/Users/romainfouilland/code/envs/all/bin/python"
 
 # Enable printing of each command
 set -x
@@ -25,13 +27,24 @@ setup() {
 }
 
 # Function to check for changes in nomenclature folder and run nomenclatures generation
-nomenclatures() {
+update_nomenclatures() {
   if git diff-index --quiet HEAD -- "$NOMENCLATURE_FOLDER"; then
     echo "No changes in $NOMENCLATURE_FOLDER, skipping nomenclatures generation..."
   else
     echo "Changes detected in $NOMENCLATURE_FOLDER, running nomenclatures generation..."
     cd ../nomenclature_parser
-    /Users/romainfouilland/code/envs/all/bin/python nomenclature_parser.py || (git stash && exit 1)
+    ${PYTHON_CMD[@]} nomenclature_parser.py || (git stash && exit 1)
+    git add .
+  fi
+}
+
+# Function to check for changes in test case sources folder and run generation of test cases json
+update_test_cases() {
+  if git diff-index --quiet HEAD -- "$TEST_CASE_SOURCES_FOLDER"; then
+    echo "No changes in $TEST_CASE_SOURCES_FOLDER, skipping test cases generation..."
+  else
+    echo "Changes detected in $TEST_CASE_SOURCES_FOLDER, running test cases generation..."
+    ${PYTHON_CMD[@]} workflow.py --stage test_case_parser || (git stash && exit 1)
     git add .
   fi
 }
@@ -44,7 +57,8 @@ run() {
   git pull --rebase
   setup
   git add ..
-  nomenclatures
+  update_nomenclatures
+  update_test_cases
   if git diff-index --quiet HEAD -- ..; then
     echo "No changes found, skipping commit."
   else

csv_parser/csv_parser.py

@@ -87,8 +87,9 @@ def get_nomenclature(elem):
         nomenclature_name = elem['Détails de format'][elem['Détails de format'].index(':')+1:].strip()
         path_file = ''
         nomenclature_files = os.listdir(os.path.join("..", "nomenclature_parser", "out", "latest", "csv"))
+        file_extension = ".csv"
         for filename in nomenclature_files:
-            if filename.startswith(nomenclature_name):
+            if filename[:-len(file_extension)] == nomenclature_name:
                 path_file = os.path.join("..", "nomenclature_parser", "out", "latest", "csv", filename)
 
         if path_file != '':
@@ -115,8 +116,8 @@ def get_nomenclature(elem):
     def is_custom_content():
         return MODEL_TYPE == "customContent"
 
-    def is_allowing_additional_properties():
-        return is_custom_content() or MODEL_TYPE == "DistributionElement"
+    def is_allowing_additional_properties(name):
+        return is_custom_content() or MODEL_TYPE == "DistributionElement" or name == "RC-DE"
 
     Path('out/' + name).mkdir(parents=True, exist_ok=True)
 
@@ -480,7 +481,7 @@ def build_example(elem):
         'required': [],
         'properties': {},
         'definitions': {},
-        'additionalProperties': is_allowing_additional_properties()
+        'additionalProperties': is_allowing_additional_properties(name)
     }
 
     def has_format_details(elem, details):

csv_parser/json_schema2xsd/src/main/java/json_schema2xsd/App.java

@@ -151,7 +151,7 @@ private static void convertRegexValues(JsonNode node) {
     }
 
     private static String getTargetNamespace(HashMap<String, String> schema) {
-        return "urn:emergency:" + (schema.get("xmlns") != null ? schema.get("xmlns") : "cisu:3.0:"+schema.get("schema"));
+        return "urn:emergency:" + (schema.get("xmlns") != null ? schema.get("xmlns") : "eda:1.9:"+schema.get("schema"));
     }
 
     /*