diff --git a/.circleci/config.yml b/.circleci/config.yml index 9cc87c0d..cda1bb9c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,80 +1,10 @@ version: 2.1 commands: - checkout-repositories: - description: "checkout the repositories and set correct publishing url" - parameters: - production_branch: - type: string - default: master - test_branch: - type: string - default: test - other_hostname: - type: string - test_hostname: - type: string - production_hostname: - type: string - steps: - - when: - condition: - equal: [ << pipeline.git.branch >>, <> ] - steps: - - run: - command: $PWD/scripts/checkoutRepositories.sh /tmp/workspace $PWD/config/publication.json $PWD/config - environment: - HOSTNAME: <> - - when: - condition: - equal: [ << pipeline.git.branch >>,<> ] - steps: - - run: - command: $PWD/scripts/checkoutRepositories.sh /tmp/workspace $PWD/config/publication.json $PWD/config - environment: - HOSTNAME: <> - - unless: - condition: - or: - - equal: [ << pipeline.git.branch >>,<> ] - - equal: [ << pipeline.git.branch >>,<> ] - steps: - - run: - command: $PWD/scripts/checkoutRepositories.sh /tmp/workspace $PWD/config/publication.json $PWD/config - environment: - HOSTNAME: <> jobs: checkout: - parameters: - production_branch: - type: string - default: master - test_branch: - type: string - default: test - production_folder: - type: string - default: production - test_folder: - type: string - default: test - other_folder: - type: string - default: dev - other_hostname: - type: string - test_hostname: - type: string - production_hostname: - type: string docker: - - image: circleci/python - environment: - PRODUCTION_BRANCH: <> - TEST_BRANCH: <> - PRODUCTION_FOLDER: <> - TEST_FOLDER: <> - OTHER_FOLDER: <> + - image: cimg/node:18.16.0 steps: - checkout - add_ssh_keys: @@ -83,19 +13,18 @@ jobs: - "0f:f5:f3:70:0a:47:30:b5:4d:c7:71:97:f9:63:9f:c9" - "f1:b9:c3:88:b2:23:a4:ab:9d:b0:b6:56:01:cb:e5:c8" - "ec:8f:5a:dc:dd:16:4f:7e:4a:ea:e5:9e:ac:f2:a1:a7" - - run: $PWD/scripts/update_sshconfig.sh private1 + - run: + name: update and add extra statements of updating the ssh config for each private ssh key + command: | + $PWD/scripts/update_sshconfig.sh private1 - run: $PWD/scripts/update_sshconfig.sh uri.semic.eu-thema - run: $PWD/scripts/update_sshconfig.sh Semicthema-DCAT-AP - run: mkdir -p workspace - run: ls -al workspace - run: - command: $PWD/scripts/findPublicationsToUpdate.sh /tmp/workspace $PWD/config/publication.json $PWD/ $PWD/config - - checkout-repositories: - production_branch: <> - test_branch: <> - other_hostname: <> - test_hostname: <> - production_hostname: <> + command: $PWD/scripts/findPublicationsToUpdate.sh /tmp/workspace $PWD/config/publication.json $PWD/ + - run: + command: $PWD/scripts/checkoutRepositories.sh /tmp/workspace $PWD/config/publication.json $PWD/config - run: if [ -e /tmp/workspace/checkouts.txt ] ; then cat /tmp/workspace/checkouts.txt ; fi - run: if [ -e /tmp/workspace/rawcheckouts.txt ] ; then cat /tmp/workspace/rawcheckouts.txt ; fi - run: if [ -e /home/circleci/project/changes.txt ] ; then cat /home/circleci/project/changes.txt ; fi @@ -157,7 +86,7 @@ jobs: [ $( egrep " ERROR " /tmp/workspace/report/*.report | wc -l ) == 0 ] || exit 1 normalise-jsonld: docker: - - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.6 + - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.9 steps: - checkout - attach_workspace: @@ -175,7 +104,7 @@ jobs: - src/ render-translation-json: docker: - - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.6 + - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.9 steps: - checkout - attach_workspace: @@ -201,7 +130,7 @@ jobs: - src/ validate-and-generate-translation-report: docker: - - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.6 + - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.9 steps: - checkout - attach_workspace: @@ -217,7 +146,7 @@ jobs: [ $( egrep " WARNING " /tmp/workspace/report/translation/*.report | wc -l ) == 0 ] || exit 1 render-merged-jsonld: docker: - - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.6 + - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.9 steps: - checkout - attach_workspace: @@ -261,7 +190,7 @@ jobs: - reporthtml/ render-example-templates: docker: - - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.6 + - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.9 steps: - checkout - attach_workspace: @@ -301,7 +230,7 @@ jobs: - target/ render-xsd-details: docker: - - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.6 + - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.9 steps: - checkout - attach_workspace: @@ -346,8 +275,6 @@ jobs: - report/ convert-json-to-ttl: docker: - #- image: circleci/ruby:2.6.5-stretch - #- image: circleci/ruby:3.0.2-buster - image: cimg/ruby:3.0.2 steps: - checkout @@ -360,6 +287,7 @@ jobs: mkdir -p /tmp/workspace/target - run: name: Convert -SHACL.jsonld file to -SHACL.ttl files + no_output_timeout: 30m command: | find /tmp/workspace/target -name \*-SHACL.jsonld -type f | $PWD/scripts/convert_to_any.sh turtle - run: @@ -373,7 +301,7 @@ jobs: - target/ expand-links: docker: - - image: circleci/node:4.8.2 + - image: cimg/node:18.16.0 steps: - checkout - attach_workspace: @@ -393,7 +321,7 @@ jobs: - target/ render-voc-jsonld: docker: - - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.7 + - image: informatievlaanderen/oslo-specification-generator:multilingual-dev-0.9 steps: - checkout - attach_workspace: @@ -433,18 +361,22 @@ jobs: touch /tmp/workspace/target/.ignore12 - run: name: Convert vocabulary to turtle + no_output_timeout: 30m command: | find /tmp/workspace/target/doc/vocabularium -name \*.jsonld -type f | $PWD/scripts/convert_to_any.sh turtle - run: name: Convert vocabulary to ntriples + no_output_timeout: 30m command: | find /tmp/workspace/target/doc/vocabularium -name \*.jsonld -type f | $PWD/scripts/convert_to_any.sh ntriples - run: name: Convert vocabulary to rdf/xml + no_output_timeout: 30m command: | find /tmp/workspace/target/doc/vocabularium -name \*.jsonld -type f | $PWD/scripts/convert_to_any.sh rdfxml - run: name: list results + no_output_timeout: 30m command: | if [ -d /tmp/workspace/doc/vocabularium ]; then ls /tmp/workspace/doc/vocabularium; fi - persist_to_workspace: @@ -453,7 +385,7 @@ jobs: - target/ copy-raw: docker: - - image: circleci/python + - image: cimg/node:18.16.0 steps: - checkout - attach_workspace: @@ -475,23 +407,29 @@ jobs: - raw/ create-artifact: parameters: - repository: + generatedReporFingerprint: type: string docker: - - image: circleci/node + - image: cimg/node:18.16.0 steps: - checkout - attach_workspace: at: /tmp/workspace - add_ssh_keys: fingerprints: + - << parameters.generatedReporFingerprint >> - "6d:07:6e:01:36:b0:89:1d:b2:93:6e:bf:86:a1:4d:61" + - run: + name: update and add extra statements of updating the ssh config for each private ssh key + command: | + export GENERATEDREPO=$(jq -r .generatedrepository.repository $PWD/config/config.json) + $PWD/scripts/update_sshconfig.sh ${GENERATEDREPO} - run: name: Checkout target repo - workdir: /tmp command: | - rm -rf generated - git clone <> generated + rm -rf /tmp/generated + export REPO=$(jq -r .generatedrepository $PWD/config/config.json) + $PWD/scripts/checkoutRepoGithub.sh "${REPO}" /tmp/generated ${TOOLCHAIN_TOKEN} - run: name: clean the generated directory workdir: /tmp/generated @@ -503,6 +441,13 @@ jobs: name: Copy static assets command: | cp -R site-skeleton/* /tmp/generated/ +# - run: +# name: deploy statistics +# command: | +# export CIRCLEWKD=$( eval echo "${CIRCLE_WORKING_DIRECTORY}" ) +# pushd ${CIRCLEWKD}/statistics +# ./deploy.sh /tmp/generated +# popd - run: name: Copy generated assets workdir: /tmp/workspace @@ -515,7 +460,7 @@ jobs: if [ -d translation ] ; then if [ ! "$(ls -A translation)" ] ; then echo "directory translation is empty" ; else cp -rT translation /tmp/generated/report/ ; fi fi if [ -d examples ] ; then if [ ! "$(ls -A examples)" ] ; then echo "directory examples is empty" ; else cp -rT examples /tmp/generated/examples/ ; fi fi - run: - name: List the files which have been created + name: check if all publication points are generated workdir: /tmp/generated command: | ls -al * @@ -540,8 +485,12 @@ jobs: name: Push results to github workdir: /tmp/generated command: | - git config user.email "contact@semic.eu" - git config user.name "Circle CI Builder" + # expand the tilde first + export CIRCLEWKD=$( eval echo "${CIRCLE_WORKING_DIRECTORY}" ) + export EMAIL=$(jq -r .generatedrepository.gitEmail $CIRCLEWKD/config/config.json) + export USER=$(jq -r .generatedrepository.gitUser $CIRCLEWKD/config/config.json) + git config user.email "${EMAIL}" + git config user.name "${USER}" git add . git status git commit -m "Applying changes from commit ${CIRCLE_SHA1}" --allow-empty @@ -553,15 +502,7 @@ workflows: version: 2 generate_documentation: jobs: - - checkout: - test_branch: test - production_branch: production - other_folder: dev - test_folder: test - production_folder: production - other_hostname: https://semiceu.github.io/ - test_hostname: https://semiceu.github.io/ - production_hostname: https://semiceu.github.io/ + - checkout - extract-jsonld-details: requires: - checkout @@ -617,6 +558,6 @@ workflows: - render-xsd-details - render-voc-rdf - create-artifact: - repository: git@github.com:SEMICeu/uri.semic.eu-generated.git + generatedReporFingerprint: "6d:07:6e:01:36:b0:89:1d:b2:93:6e:bf:86:a1:4d:61" requires: - expand-links diff --git a/config/config.json b/config/config.json index eec048a0..ac0c1e80 100644 --- a/config/config.json +++ b/config/config.json @@ -3,18 +3,27 @@ "otherLanguages": [ "fr", "de" ], - "hostname": "https://semiceu.github.io/", + "hostname": "https://semiceu.github.io", "domain": "data.europa.eu", "publicationpoints" : ["dev"], "generatedrepository" : { "organisation": "SEMICeu", "repository" : "uri.semic.eu-generated", - "private" : true + "private" : true, + "gitEmail" : "contact@semic.eu", + "gitUser" : "Circle CI Builder" }, - "toolchainversion" : "3.0.5", + "toolchainversion" : "3.1.1", "toolchain" : { "strickness" : "lazy", "version": 3, - "triggerall" : false + "triggerall" : true + }, + "shaclgenerator" : { + "parameters": "-m individual -c 'uniqueLanguages' -c 'nodeKind'", + "parameters-old": "" + }, + "jsonldgenerator" : { + "parameters" : "-d -l label" } } diff --git a/config/context b/config/context index 480c31ea..4318ae9f 100644 --- a/config/context +++ b/config/context @@ -90,7 +90,7 @@ "@type" : "@id" }, "usage": { - "@id": "vann:usageNote", + "@id": "skos:scopeNote", "@container": "@language" }, "parents" :{ diff --git a/config/dev/publication.json b/config/dev/publication.json index c45ceeb8..2f6f2407 100755 --- a/config/dev/publication.json +++ b/config/dev/publication.json @@ -161,16 +161,16 @@ "navigation": { "prev": "https://semiceu.github.io/Core-Location-Vocabulary/releases/2.0.2" }, - "dummy": "75" + "dummy": "80" }, { "urlref": "/doc/vocabularium/core-location", "repository": "git@uri.semic.eu-thema:SEMICeu/uri.semic.eu-thema.git", - "branchtag": "main", + "branchtag": "ReSpecCV", "name": "core-location", "filename": "config/core-location-voc.json", "navigation": {}, - "dummy": "23" + "dummy": "24" }, { "dummy": "8", diff --git a/scripts/README.md b/scripts/README.md index f850ab60..ccbbb505 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -6,8 +6,8 @@ The scripts are so designed that all configuration is present in the config dire Each script has as implicit requirement that the execution environment is the one in which is called as defined in CIRCLECI setup. Executing the scripts thus cannot be done directly after checking out this repository. -One has to create an execution environment (a Docker container) statisfying the needs. -This can be done by mannualy executing the CIRCLECI configuration. +One has to create an execution environment (a Docker container) satisfying the needs. +This can be done by manually executing the CIRCLECI configuration. diff --git a/scripts/checkoutRepoGithub.sh b/scripts/checkoutRepoGithub.sh new file mode 100755 index 00000000..2103d23d --- /dev/null +++ b/scripts/checkoutRepoGithub.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +set -x + +# checkout a full repository from GitHub +# this takes as input a json description of the file +# to support cloning a public repository aswell a private repository +# +# arg1 = the description of the file +# arg2 = the target where the downloaded file has to be written +# arg3 = the PAT from a user having access to the repository +# +# for more information on PAT (Personal Access Token) see +# https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens + +# The input json structure is of the form +# { +# "organisation" : "", +# "repository" : "", +# "filepath" : "", +# "branchtag" : "", +# "private" : boolean +# } + +FILEDESC=$1 +TARGET=$2 +TOKEN=$3 + +#-----------------------------------------------------------------------------# +# +TMPDIR=/tmp + +# clean +rm -f ${TMPDIR}/apifile.out +rm -f ${TMPDIR}/file.out + +rm -rf ${TARGET} +mkdir -p ${TARGET} + + +checkHTTPcode () { + # HTTP code must be between 200 and 300, otherwise script fails + if [ "$1" -lt "200" ] || [ "$1" -ge "300" ] ; then + echo "Access unsuccessful, possible new token should be inserted" + exit 1 + fi +} + +checkHTTPcodeContinue () { + # HTTP code must be between 200 and 300, otherwise script fails + if [ "$1" -lt "200" ] || [ "$1" -ge "300" ] ; then + echo "Access unsuccessful, possible new token should be inserted" + fi +} + +#-----------------------------------------------------------------------------# +# +# echo $FILEDESC | jq . + +ORG=`echo ${FILEDESC} | jq -r .organisation` +REPO=`echo ${FILEDESC} | jq -r .repository` +FILEPATH=`echo ${FILEDESC} | jq -r .filepath` +BRANCHTAG=`echo ${FILEDESC} | jq -r .branchtag` +PRIVATE=`echo ${FILEDESC} | jq -r .private` + + +if [ "${TOKEN}" != "" ] ; then + echo "cloning private repository" + STATUS=`curl -s -w "%{http_code}" -H "Authorization: token ${TOKEN}" -o ${TMPDIR}/apifile.out https://api.github.com/repos/${ORG}/${REPO}` + checkHTTPcode ${STATUS} + CLONEURL=`jq -r .ssh_url ${TMPDIR}/apifile.out` + git clone ${CLONEURL} ${TARGET} +else + echo "cloning public repository" + if [ "${PRIVATE}" == "true" ] & [ "${TOKEN}" == "" ] ; then + echo "WARNING: trying to clone a private repo without token" + STATUS=`curl -s -w "%{http_code}" -o ${TMPDIR}/apifile.out https://api.github.com/repos/${ORG}/${REPO}` + checkHTTPcodeContinue ${STATUS} + echo "WARNING: try clone a private repo using ssh" + CLONEURL="git@github.com:${ORG}/${REPO}.git" + git clone ${CLONEURL} ${TARGET} + else + STATUS=`curl -s -w "%{http_code}" -o ${TMPDIR}/apifile.out https://api.github.com/repos/${ORG}/${REPO}` + checkHTTPcode ${STATUS} + CLONEURL=`jq -r .ssh_url ${TMPDIR}/apifile.out` + git clone ${CLONEURL} ${TARGET} + fi +fi + + + + + + + diff --git a/scripts/checkoutRepositories.sh b/scripts/checkoutRepositories.sh index 1406c672..cb49dbc7 100755 --- a/scripts/checkoutRepositories.sh +++ b/scripts/checkoutRepositories.sh @@ -40,6 +40,7 @@ HOSTNAME=$(jq -r .hostname ${CONFIGDIR}/config.json) cleanup_directory() { rm -rf .git + rm -rf codelijsten local MAPPINGFILE=`jq -r 'if (.filename | length) > 0 then .filename else @sh "config/eap-mapping.json" end' .publication-point.json` if [[ -f ".names.txt" && -f $MAPPINGFILE ]] @@ -47,6 +48,8 @@ cleanup_directory() { STR=".[] | select(.name == \"$(cat .names.txt)\") | [.]" jq "${STR}" ${MAPPINGFILE} >.map.json jq -r '.[] | @sh "find . -name \"*.eap\" ! -name \(.eap) -type f -exec rm -f {} + "' .map.json | bash -e + SITE=`jq -r .[].site .map.json` + find ./site-skeleton -depth -type d ! -wholename "./site-skeleton" ! -wholename "./${SITE}" -exec rm -rf {} + fi } diff --git a/scripts/findPublicationsToUpdate.sh b/scripts/findPublicationsToUpdate.sh index 3cf6a0a5..5090cdba 100755 --- a/scripts/findPublicationsToUpdate.sh +++ b/scripts/findPublicationsToUpdate.sh @@ -123,6 +123,10 @@ if [[ $changesRequireBuild == "true" && $onlyChangedPublicationFiles == "true" ] else onlyChangedPublications=true fi + if [[ ${TRIGGERALL} == "false" ]]; then + echo "WARNING: a full rebuild is switched off. " + onlyChangedPublications=true + fi else if [[ ${TRIGGERALL} == "false" ]]; then echo "WARNING: a full rebuild is switched off. " diff --git a/scripts/render-details.sh b/scripts/render-details.sh index c90b0126..5656ede3 100755 --- a/scripts/render-details.sh +++ b/scripts/render-details.sh @@ -23,6 +23,29 @@ execution_strickness() { fi } +generator_parameters() { + + local GENERATOR=$1 + local JSONI=$2 + + # + # The toolchain can add specific parameters for the SHACL generation tool + # Priority rules are as follows: + # 1. publication point specific + # 2. generic configuration + # 3. otherwise empty string + # + COMMAND=$(echo '.'${GENERATOR}'.parameters' ) + PARAMETERS=$(jq -r '.shaclgenerator.parameters' ${JSONI}) + if [ "${PARAMETERS}" == "null" ] ; then + PARAMETERS=$(jq -r '.shaclgenerator.parameters ' ${CONFIGDIR}/config.json) + fi + if [ "${PARAMETERS}" == "null" ] || [ -z "${PARAMETERS}" ] ; then + PARAMETERS="" + fi +} + + render_merged_files() { echo "Rendering the merged version of $1 with the json in $2 from $3 and to $4" local JSONI=$1 @@ -245,8 +268,11 @@ render_context() { # SLINE TLINE JSON COMMAND=$(echo '.[]|select(.name | contains("'${BASENAME}'"))|.type') TYPE=$(jq -r "${COMMAND}" ${SLINE}/.names.json) + + generator_parameters contextgenerator ${JSONI} + if [ ${TYPE} == "ap" ] || [ ${TYPE} == "oj" ]; then - echo "RENDER-DETAILS(context): node /app/json-ld-generator2.js -c -l label -i ${JSONI} -o ${TLINE}/context/${OUTFILELANGUAGE} " + echo "RENDER-DETAILS(context): node /app/json-ld-generator.js ${PARAMETERS} -i ${JSONI} -o ${TLINE}/context/${OUTFILELANGUAGE} " mkdir -p ${TLINE}/context COMMANDJSONLD=$(echo '.[].translation | .[] | select(.language | contains("'${GOALLANGUAGE}'")) | .mergefile') LANGUAGEFILENAMEJSONLD=$(jq -r "${COMMANDJSONLD}" ${SLINE}/.names.json) @@ -256,8 +282,8 @@ render_context() { # SLINE TLINE JSON MERGEDJSONLD=${RLINE}/translation/${LANGUAGEFILENAMEJSONLD} - echo "RENDER-DETAILS(context-language-aware): node /app/json-ld-generator2.js -c -l label -i ${MERGEDJSONLD} -o ${TLINE}/context/${OUTFILELANGUAGE} -m ${GOALLANGUAGE}" - if ! node /app/json-ld-generator2.js -c -l label -i ${MERGEDJSONLD} -o ${TLINE}/context/${OUTFILELANGUAGE} -m ${GOALLANGUAGE}; then + echo "RENDER-DETAILS(context-language-aware): node /app/json-ld-generator2.js -d -l label -i ${MERGEDJSONLD} -o ${TLINE}/context/${OUTFILELANGUAGE} -m ${GOALLANGUAGE}" + if ! node /app/json-ld-generator2.js -d -l label -i ${MERGEDJSONLD} -o ${TLINE}/context/${OUTFILELANGUAGE} -m ${GOALLANGUAGE}; then echo "RENDER-DETAILS(context-language-aware): See XXX for more details, Rendering failed" execution_strickness else @@ -316,6 +342,8 @@ render_shacl_languageaware() { local GOALLANGUAGE=$6 local PRIMELANGUAGE=${7-false} + + FILENAME=$(jq -r ".name" ${JSONI}) COMMANDJSONLD=$(echo '.[].translation | .[] | select(.language | contains("'${GOALLANGUAGE}'")) | .mergefile') LANGUAGEFILENAMEJSONLD=$(jq -r "${COMMANDJSONLD}" ${SLINE}/.names.json) @@ -333,13 +361,16 @@ render_shacl_languageaware() { COMMAND=$(echo '.[]|select(.name | contains("'${BASENAME}'"))|.type') TYPE=$(jq -r "${COMMAND}" ${SLINE}/.names.json) + + generator_parameters shaclgenerator ${JSONI} + if [ ${TYPE} == "ap" ] || [ ${TYPE} == "oj" ]; then DOMAIN="${HOSTNAME}/${LINE}" - echo "RENDER-DETAILS(shacl-languageaware): node /app/shacl-generator.js -i ${MERGEDJSONLD} -m individual -c 'uniqueLanguages' -c 'nodekind' -d ${DOMAIN} -p ${DOMAIN} -o ${OUTFILE} -l ${GOALLANGUAGE} -a" + echo "RENDER-DETAILS(shacl-languageaware): node /app/shacl-generator.js -i ${MERGEDJSONLD} ${PARAMETERS} -d ${DOMAIN} -p ${DOMAIN} -o ${OUTFILE} -l ${GOALLANGUAGE}" pushd /app mkdir -p ${TLINE}/shacl mkdir -p ${RLINE}/shacl - if ! node /app/shacl-generator2.js -i ${MERGEDJSONLD} -m individual -c 'uniqueLanguages' -c 'nodekind' -d ${DOMAIN} -p ${DOMAIN} -o ${OUTFILE} -l ${GOALLANGUAGE} -a --labelAsName 2>&1 | tee ${OUTREPORT}; then + if ! node /app/shacl-generator2.js -i ${MERGEDJSONLD} ${PARAMETERS} -d ${DOMAIN} -p ${DOMAIN} -o ${OUTFILE} -l ${GOALLANGUAGE} 2>&1 | tee ${OUTREPORT}; then echo "RENDER-DETAILS(shacl-languageaware): See ${OUTREPORT} for the details" execution_strickness else @@ -401,55 +432,6 @@ render_xsd() { # SLINE TLINE JSON fi } -# -# if xsd rendering is final, then this function can replace the old one. -render_new_xsd() { # SLINE TLINE JSON - echo "render_xsd: $1 $2 $3 $4 $5" - local SLINE=$1 - local TLINE=$2 - local JSONI=$3 - local RLINE=$4 - local GOALLANGUAGE=$5 - local PRIMELANGUAGE=${6-false} - - FILENAME=$(jq -r ".name" ${JSONI}) - OUTFILE=${FILENAME}.xsd - OUTFILELANGUAGE=${FILENAME}_${GOALLANGUAGE}.xsd - - BASENAME=$(basename ${JSONI} .jsonld) - - COMMAND=$(echo '.[]|select(.name | contains("'${BASENAME}'"))|.type') - TYPE=$(jq -r "${COMMAND}" ${SLINE}/.names.json) - - XSDDOMAIN="https://data.europa.eu/m8g/xml/" - - if [ ${TYPE} == "ap" ] || [ ${TYPE} == "oj" ]; then - - mkdir -p ${TLINE}/xsd - COMMANDJSONLD=$(echo '.[].translation | .[] | select(.language | contains("'${GOALLANGUAGE}'")) | .mergefile') - LANGUAGEFILENAMEJSONLD=$(jq -r "${COMMANDJSONLD}" ${SLINE}/.names.json) - if [ "${LANGUAGEFILENAMEJSONLD}" == "" ] ; then - echo "configuration for language ${GOALLANGUAGE} not present. Ignore this language for ${SLINE}" - else - - MERGEDJSONLD=${RLINE}/translation/${LANGUAGEFILENAMEJSONLD} - - echo "RENDER-DETAILS(xsd): node /app/xsd-generator.js -d -l label -i ${MERGEDJSONLD} -o ${TLINE}/xsd/${OUTFILELANGUAGE} -m ${GOALLANGUAGE} -b ${XSDDOMAIN}" - if ! node /app/new-xsd-generator.js -d -l label -i ${MERGEDJSONLD} -o ${TLINE}/xsd/${OUTFILELANGUAGE} -m ${GOALLANGUAGE} -b ${XSDDOMAIN}; then - echo "RENDER-DETAILS(xsd): See XXX for more details, Rendering failed" - execution_strickness - else - echo "RENDER-DETAILS(xsd): Rendering successfull, File saved to ${TLINE}/xsd/${OUTFILELANGUAGE}" - fi - - if [ ${PRIMELANGUAGE} == true ] ; then - cp ${TLINE}/xsd/${OUTFILELANGUAGE} ${TLINE}/xsd/${OUTFILE} - fi - - fi - fi -} - write_report() { echo "Rendering the reportfiles of $1 with the json in $2 from $3 and to $4" local JSONI=$1 @@ -514,10 +496,10 @@ cat ${CHECKOUTFILE} | while read line; do done ;; xsd) - render_new_xsd $SLINE $TLINE $i $RLINE ${PRIMELANGUAGE} true + render_xsd $SLINE $TLINE $i $RLINE ${PRIMELANGUAGE} true for g in ${GOALLANGUAGE} do - render_new_xsd $SLINE $TLINE $i $RLINE ${g} + render_xsd $SLINE $TLINE $i $RLINE ${g} done ;; multilingual) diff --git a/scripts/update_sshconfig.sh b/scripts/update_sshconfig.sh index 7fa217b8..4a9c3e79 100755 --- a/scripts/update_sshconfig.sh +++ b/scripts/update_sshconfig.sh @@ -26,7 +26,7 @@ # arg1 the that should be updated - +${CIRCLECIHOSTNAME}=$1 sed -i -e "/Host ${CIRCLECIHOSTNAME}/a \ \ User git" ~/.ssh/config sed -i -e "/Host ${CIRCLECIHOSTNAME}/a \ \ HostName github.com" ~/.ssh/config