From 0a39cabc7cf038efff78dea4407708cf073419ac Mon Sep 17 00:00:00 2001 From: michimau Date: Mon, 22 Aug 2022 14:33:42 +0200 Subject: [PATCH 01/18] build in docker, with multistage approach --- build-in-docker/Dockerfile | 39 +++++++ build-in-docker/build_last_image.sh | 10 ++ build-in-docker/docker-entrypoint.sh | 153 +++++++++++++++++++++++++++ 3 files changed, 202 insertions(+) create mode 100644 build-in-docker/Dockerfile create mode 100755 build-in-docker/build_last_image.sh create mode 100644 build-in-docker/docker-entrypoint.sh diff --git a/build-in-docker/Dockerfile b/build-in-docker/Dockerfile new file mode 100644 index 0000000000..be9baaf196 --- /dev/null +++ b/build-in-docker/Dockerfile @@ -0,0 +1,39 @@ +FROM jetty:9-jdk8-openjdk as build + +USER root +RUN apt-get -y update && \ + apt-get install -y maven + +USER jetty +RUN git clone --recursive https://github.com/eea/geonetwork-eea.git + +#RUN git clone --recursive https://github.com/eea/geonetwork-eea.git 2> /dev/null || (cd geonetwork-eea ; git pull) +RUN cd geonetwork-eea && \ + mvn clean install -DskipTests +RUN cd geonetwork-eea/web && \ + mvn package -DskipTests -Penv-catalogue + + + +FROM jetty:9-jre8-openjdk +MAINTAINER michimau + +ARG id +USER root + +RUN rm -rf /var/lib/jetty/webapps/* && \ + chown jetty:jetty /var/lib/jetty/webapps + +COPY --from=build /var/lib/jetty/geonetwork-eea/web/target/catalogue.war /var/lib/jetty/webapps/ + +RUN rm -rf /var/lib/jetty/webapps/* && \ + chown jetty:jetty /var/lib/jetty/webapps + +COPY docker-entrypoint.sh / + +RUN mkdir -p /var/local/gn_data && \ + chown -R jetty:jetty /var/local/gn_data + +USER jetty + +ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/build-in-docker/build_last_image.sh b/build-in-docker/build_last_image.sh new file mode 100755 index 0000000000..7a4ed313e4 --- /dev/null +++ b/build-in-docker/build_last_image.sh @@ -0,0 +1,10 @@ +#!/bin/bash + + + + + +git ls-remote https://github.com/eea/geonetwork-eea.git HEAD | cut -f1 + + +docker build -t michimau/gn:$(git ls-remote https://github.com/eea/geonetwork-eea.git HEAD | cut -f1) . diff --git a/build-in-docker/docker-entrypoint.sh b/build-in-docker/docker-entrypoint.sh new file mode 100644 index 0000000000..4523232a2a --- /dev/null +++ b/build-in-docker/docker-entrypoint.sh @@ -0,0 +1,153 @@ +#!/bin/sh + +set -e + +mkdir -p /catalogue-data/htmlcache +mkdir -p /catalogue-data/lucene + +unzip -o /target/$CATALOGUE.war -d /var/lib/jetty/webapps/$CATALOGUE + +if [ "${ES_HOST}" != "localhost" ]; then + sed -i "s#http://localhost:9200#${ES_PROTOCOL:="http"}://${ES_HOST}:${ES_PORT:="9200"}#g" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/web.xml" ; + sed -i "s#es.host=localhost#es.host=${ES_HOST}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; +fi; + +if [ -n "${ES_PROTOCOL}" ] && [ "${ES_PROTOCOL}" != "http" ] ; then + sed -i "s#es.protocol=http#es.protocol=${ES_PROTOCOL}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; +fi + +if [ -n "${ES_PORT}" ] && [ "$ES_PORT" != "9200" ] ; then + sed -i "s#es.port=9200#es.port=${ES_PORT}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; +fi + +if [ -n "${ES_INDEX_RECORDS}" ] && [ "$ES_INDEX_RECORDS" != "gn-records" ] ; then + sed -i "s#es.index.records=gn-records#es.index.records=${ES_INDEX_RECORDS}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; +fi + +if [ "${ES_USERNAME}" != "" ] ; then + sed -i "s#es.username=#es.username=${ES_USERNAME}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; +fi + +if [ "${ES_PASSWORD}" != "" ] ; then + sed -i "s#es.password=#es.password=${ES_PASSWORD}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; +fi + +if [ -n "${KB_URL}" ] && [ "$KB_URL" != "http://localhost:5601" ]; then + sed -i "s#kb.url=http://localhost:5601#kb.url=${KB_URL}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; + sed -i "s#http://localhost:5601#${KB_URL}#g" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/web.xml" ; +fi + +if [ "$1" = jetty.sh ]; then + if ! command -v bash >/dev/null 2>&1 ; then + cat >&2 <<- 'EOWARN' + ******************************************************************** + ERROR: bash not found. Use of jetty.sh requires bash. + ******************************************************************** + EOWARN + exit 1 + fi + cat >&2 <<- 'EOWARN' + + ******************************************************************** + WARNING: Use of jetty.sh from this image is deprecated and may + be removed at some point in the future. + + See the documentation for guidance on extending this image: + https://github.com/docker-library/docs/tree/master/jetty + ******************************************************************** + EOWARN +fi + +if ! command -v -- "$1" >/dev/null 2>&1 ; then + set -- java -jar "$JETTY_HOME/start.jar" "$@" +fi + +: ${TMPDIR:=/tmp/jetty} +[ -d "$TMPDIR" ] || mkdir -p $TMPDIR 2>/dev/null + +: ${JETTY_START:=$JETTY_BASE/jetty.start} + +case "$JAVA_OPTIONS" in + *-Djava.io.tmpdir=*) ;; + *) JAVA_OPTIONS="-Djava.io.tmpdir=$TMPDIR $JAVA_OPTIONS" ;; +esac + +if expr "$*" : 'java .*/start\.jar.*$' >/dev/null ; then + # this is a command to run jetty + + # check if it is a terminating command + for A in "$@" ; do + case $A in + --add-to-start* |\ + --create-files |\ + --create-startd |\ + --download |\ + --dry-run |\ + --exec-print |\ + --help |\ + --info |\ + --list-all-modules |\ + --list-classpath |\ + --list-config |\ + --list-modules* |\ + --stop |\ + --update-ini |\ + --version |\ + -v )\ + # It is a terminating command, so exec directly + JAVA="$1" + shift + exec $JAVA $JAVA_OPTIONS "$@" + esac + done + + if [ $(whoami) != "jetty" ]; then + cat >&2 <<- EOWARN + ******************************************************************** + WARNING: User is $(whoami) + The user should be (re)set to 'jetty' in the Dockerfile + ******************************************************************** + EOWARN + fi + + if [ -f $JETTY_START ] ; then + if [ $JETTY_BASE/start.d -nt $JETTY_START ] ; then + cat >&2 <<- EOWARN + ******************************************************************** + WARNING: The $JETTY_BASE/start.d directory has been modified since + the $JETTY_START files was generated. Either delete + the $JETTY_START file or re-run + /generate-jetty.start.sh + from a Dockerfile + ******************************************************************** + EOWARN + fi + echo $(date +'%Y-%m-%d %H:%M:%S.000'):INFO:docker-entrypoint:jetty start from $JETTY_START + set -- $(cat $JETTY_START) + else + # Do a jetty dry run to set the final command + JAVA="$1" + shift + $JAVA $JAVA_OPTIONS "$@" --dry-run > $JETTY_START + if [ $(egrep -v '\\$' $JETTY_START | wc -l ) -gt 1 ] ; then + # command was more than a dry-run + cat $JETTY_START \ + | awk '/\\$/ { printf "%s", substr($0, 1, length($0)-1); next } 1' \ + | egrep -v '[^ ]*java .* org\.eclipse\.jetty\.xml\.XmlConfiguration ' + exit + fi + set -- $(sed -e 's/ -Djava.io.tmpdir=[^ ]*//g' -e 's/\\$//' $JETTY_START) + fi +fi + +if [ "${1##*/}" = java -a -n "$JAVA_OPTIONS" ] ; then + JAVA="$1" + shift + set -- "$JAVA" $JAVA_OPTIONS "$@" +fi + +exec "$@" + + + + From b6bb7ed857916881792ec9b9a1816f4a62b6cb08 Mon Sep 17 00:00:00 2001 From: michimau Date: Mon, 22 Aug 2022 14:39:46 +0200 Subject: [PATCH 02/18] rename of the image name --- build-in-docker/build_last_image.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/build-in-docker/build_last_image.sh b/build-in-docker/build_last_image.sh index 7a4ed313e4..e7c082b1c8 100755 --- a/build-in-docker/build_last_image.sh +++ b/build-in-docker/build_last_image.sh @@ -1,10 +1,3 @@ #!/bin/bash - - - - -git ls-remote https://github.com/eea/geonetwork-eea.git HEAD | cut -f1 - - -docker build -t michimau/gn:$(git ls-remote https://github.com/eea/geonetwork-eea.git HEAD | cut -f1) . +docker build -t eeacms/geonetwork-eea:$(git ls-remote https://github.com/eea/geonetwork-eea.git HEAD | cut -f1) . From 7af487eb3f1b34fe07227c903788c941c307a94c Mon Sep 17 00:00:00 2001 From: michimau Date: Thu, 25 Aug 2022 11:31:42 +0200 Subject: [PATCH 03/18] addressing remarks issued on the pull request --- build-in-docker/Dockerfile | 25 ++++++++++++++----------- build-in-docker/docker-entrypoint.sh | 7 ------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/build-in-docker/Dockerfile b/build-in-docker/Dockerfile index be9baaf196..5f2a8175cb 100644 --- a/build-in-docker/Dockerfile +++ b/build-in-docker/Dockerfile @@ -1,33 +1,32 @@ -FROM jetty:9-jdk8-openjdk as build - +FROM jetty:9-jdk8-eclipse-temurin as build + USER root RUN apt-get -y update && \ - apt-get install -y maven + apt-get install -y maven \ + git \ + zip USER jetty RUN git clone --recursive https://github.com/eea/geonetwork-eea.git -#RUN git clone --recursive https://github.com/eea/geonetwork-eea.git 2> /dev/null || (cd geonetwork-eea ; git pull) RUN cd geonetwork-eea && \ mvn clean install -DskipTests -RUN cd geonetwork-eea/web && \ +RUN cd geonetwork-eea/web && \ mvn package -DskipTests -Penv-catalogue +RUN unzip /var/lib/jetty/geonetwork-eea/web/target/catalogue.war -d /var/lib/jetty/webapps/catalogue -FROM jetty:9-jre8-openjdk +FROM jetty:9-jdk8-eclipse-temurin MAINTAINER michimau -ARG id USER root RUN rm -rf /var/lib/jetty/webapps/* && \ chown jetty:jetty /var/lib/jetty/webapps -COPY --from=build /var/lib/jetty/geonetwork-eea/web/target/catalogue.war /var/lib/jetty/webapps/ - -RUN rm -rf /var/lib/jetty/webapps/* && \ - chown jetty:jetty /var/lib/jetty/webapps +COPY --from=build /var/lib/jetty/webapps/catalogue /var/lib/jetty/webapps/catalogue +#RUN unzip /tmp/catalogue.war -d /var/lib/jetty/webapps/catalogue COPY docker-entrypoint.sh / @@ -37,3 +36,7 @@ RUN mkdir -p /var/local/gn_data && \ USER jetty ENTRYPOINT ["/docker-entrypoint.sh"] + + + + diff --git a/build-in-docker/docker-entrypoint.sh b/build-in-docker/docker-entrypoint.sh index 4523232a2a..d55f0d0873 100644 --- a/build-in-docker/docker-entrypoint.sh +++ b/build-in-docker/docker-entrypoint.sh @@ -5,8 +5,6 @@ set -e mkdir -p /catalogue-data/htmlcache mkdir -p /catalogue-data/lucene -unzip -o /target/$CATALOGUE.war -d /var/lib/jetty/webapps/$CATALOGUE - if [ "${ES_HOST}" != "localhost" ]; then sed -i "s#http://localhost:9200#${ES_PROTOCOL:="http"}://${ES_HOST}:${ES_PORT:="9200"}#g" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/web.xml" ; sed -i "s#es.host=localhost#es.host=${ES_HOST}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; @@ -47,7 +45,6 @@ if [ "$1" = jetty.sh ]; then exit 1 fi cat >&2 <<- 'EOWARN' - ******************************************************************** WARNING: Use of jetty.sh from this image is deprecated and may be removed at some point in the future. @@ -147,7 +144,3 @@ if [ "${1##*/}" = java -a -n "$JAVA_OPTIONS" ] ; then fi exec "$@" - - - - From 7adae790e1604fec5cda11e53da83b57d05420bc Mon Sep 17 00:00:00 2001 From: michimau Date: Thu, 25 Aug 2022 11:33:48 +0200 Subject: [PATCH 04/18] remove commented line --- build-in-docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/build-in-docker/Dockerfile b/build-in-docker/Dockerfile index 5f2a8175cb..22445cfc8a 100644 --- a/build-in-docker/Dockerfile +++ b/build-in-docker/Dockerfile @@ -26,7 +26,6 @@ RUN rm -rf /var/lib/jetty/webapps/* && \ chown jetty:jetty /var/lib/jetty/webapps COPY --from=build /var/lib/jetty/webapps/catalogue /var/lib/jetty/webapps/catalogue -#RUN unzip /tmp/catalogue.war -d /var/lib/jetty/webapps/catalogue COPY docker-entrypoint.sh / From eabeed671fda2c588fa7a6ea4574cd6fe845a017 Mon Sep 17 00:00:00 2001 From: michimau Date: Thu, 25 Aug 2022 11:51:31 +0200 Subject: [PATCH 05/18] adjusting pom file to streamline the data directory --- web/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/pom.xml b/web/pom.xml index 38677ba75d..d48ece0c65 100644 --- a/web/pom.xml +++ b/web/pom.xml @@ -1278,7 +1278,7 @@ prod config-securitycatalogue catalogue - /var/local/gn_data/logs/catalogue.log + /catalogue-data/logs/catalogue.log file:///catalogue-data config-security-noldap.xml From 7ed9e9d42fa8bcb4ad922fd6d9ab0e3d8aebcc84 Mon Sep 17 00:00:00 2001 From: michimau Date: Thu, 25 Aug 2022 11:55:31 +0200 Subject: [PATCH 06/18] minor refinements --- build-in-docker/Dockerfile | 7 +++---- build-in-docker/docker-entrypoint.sh | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build-in-docker/Dockerfile b/build-in-docker/Dockerfile index 22445cfc8a..35d28ea6d3 100644 --- a/build-in-docker/Dockerfile +++ b/build-in-docker/Dockerfile @@ -23,15 +23,14 @@ MAINTAINER michimau USER root RUN rm -rf /var/lib/jetty/webapps/* && \ - chown jetty:jetty /var/lib/jetty/webapps + chown jetty:jetty /var/lib/jetty/webapps && \ + mkdir -p /catalogue-data && \ + chown -R jetty:jetty /catalogue-data COPY --from=build /var/lib/jetty/webapps/catalogue /var/lib/jetty/webapps/catalogue COPY docker-entrypoint.sh / -RUN mkdir -p /var/local/gn_data && \ - chown -R jetty:jetty /var/local/gn_data - USER jetty ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/build-in-docker/docker-entrypoint.sh b/build-in-docker/docker-entrypoint.sh index d55f0d0873..63225ca2ba 100644 --- a/build-in-docker/docker-entrypoint.sh +++ b/build-in-docker/docker-entrypoint.sh @@ -4,6 +4,7 @@ set -e mkdir -p /catalogue-data/htmlcache mkdir -p /catalogue-data/lucene +mkdir -p /catalogue-data/logs if [ "${ES_HOST}" != "localhost" ]; then sed -i "s#http://localhost:9200#${ES_PROTOCOL:="http"}://${ES_HOST}:${ES_PORT:="9200"}#g" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/web.xml" ; From 35469f6e11190ac71cc1734156d3c96437d7deb8 Mon Sep 17 00:00:00 2001 From: michimau Date: Thu, 25 Aug 2022 11:58:57 +0200 Subject: [PATCH 07/18] docker-compose.sh made executable --- build-in-docker/docker-entrypoint.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 build-in-docker/docker-entrypoint.sh diff --git a/build-in-docker/docker-entrypoint.sh b/build-in-docker/docker-entrypoint.sh old mode 100644 new mode 100755 From 8177f1e96d7c339f7e7c6142078dd64168ffbfaf Mon Sep 17 00:00:00 2001 From: valentinab25 <30239069+valentinab25@users.noreply.github.com> Date: Mon, 5 Sep 2022 15:15:35 +0300 Subject: [PATCH 08/18] add Jenkinsfile to build on commit id --- Jenkinsfile | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000000..07ec2745dc --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,56 @@ +pipeline { + agent { + node { label "docker-host" } + } + + environment { + GIT_NAME = "geonetwork-eea" + SONARQUBE_TAGS = "cr.eionet.europa.eu" + registry = "eeacms/eea-geonetwork" + } + + stages { + + stage ('Docker build and push') { + when { + environment name: 'CHANGE_ID', value: '' + } + steps { + script{ + sh '''env''' + if (env.BRANCH_NAME == 'michimau-eea-4.2.0') { + tagName = '$GIT_COMMIT' + } else { + tagName = "$BRANCH_NAME" + } + try { + dockerImage = docker.build("$registry:$tagName", "--no-cache ./build-in-docker/") + docker.withRegistry( '', 'eeajenkins' ) { + dockerImage.push() + } + } + finally { + sh "docker rmi $registry:$tagName" + } + } + } + } + + } + + post { + always { + cleanWs(cleanWhenAborted: true, cleanWhenFailure: true, cleanWhenNotBuilt: true, cleanWhenSuccess: true, cleanWhenUnstable: true, deleteDirs: true) + script { + def url = "${env.BUILD_URL}/display/redirect" + def status = currentBuild.currentResult + def subject = "${status}: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'" + def summary = "${subject} (${url})" + def details = """

${env.JOB_NAME} - Build #${env.BUILD_NUMBER} - ${status}

+

Check console output at ${env.JOB_BASE_NAME} - #${env.BUILD_NUMBER}

+ """ + emailext (subject: '$DEFAULT_SUBJECT', to: '$DEFAULT_RECIPIENTS', body: details, recipientProviders: [[$class: 'DevelopersRecipientProvider'], [$class: 'CulpritsRecipientProvider']]) + } + } + } +} From b65db6517a49a279fe1ebe79b97633751ee25352 Mon Sep 17 00:00:00 2001 From: valentinab25 <30239069+valentinab25@users.noreply.github.com> Date: Mon, 5 Sep 2022 15:20:04 +0300 Subject: [PATCH 09/18] truncate tag --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 07ec2745dc..782bca30aa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,7 +19,7 @@ pipeline { script{ sh '''env''' if (env.BRANCH_NAME == 'michimau-eea-4.2.0') { - tagName = '$GIT_COMMIT' + tagName = GIT_COMMIT.take(8) } else { tagName = "$BRANCH_NAME" } From 1f0b5822cac0ee030ed16d552b62cb5b3073c721 Mon Sep 17 00:00:00 2001 From: valentinab25 <30239069+valentinab25@users.noreply.github.com> Date: Mon, 5 Sep 2022 17:29:34 +0300 Subject: [PATCH 10/18] configure to eea-4.2.0 branch --- Jenkinsfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 782bca30aa..1e163f9f2a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,6 @@ pipeline { environment { GIT_NAME = "geonetwork-eea" - SONARQUBE_TAGS = "cr.eionet.europa.eu" registry = "eeacms/eea-geonetwork" } @@ -17,8 +16,7 @@ pipeline { } steps { script{ - sh '''env''' - if (env.BRANCH_NAME == 'michimau-eea-4.2.0') { + if (env.BRANCH_NAME == 'eea-4.2.0') { tagName = GIT_COMMIT.take(8) } else { tagName = "$BRANCH_NAME" @@ -49,7 +47,7 @@ pipeline { def details = """

${env.JOB_NAME} - Build #${env.BUILD_NUMBER} - ${status}

Check console output at ${env.JOB_BASE_NAME} - #${env.BUILD_NUMBER}

""" - emailext (subject: '$DEFAULT_SUBJECT', to: '$DEFAULT_RECIPIENTS', body: details, recipientProviders: [[$class: 'DevelopersRecipientProvider'], [$class: 'CulpritsRecipientProvider']]) + emailext (subject: summary, body: details, attachLog: true, compressLog: true, recipientProviders: [[$class: 'DevelopersRecipientProvider'], [$class: 'CulpritsRecipientProvider']]) } } } From 936f62736f376bd4545dd5e2e41f0a473466af26 Mon Sep 17 00:00:00 2001 From: valentinab25 <30239069+valentinab25@users.noreply.github.com> Date: Mon, 5 Sep 2022 17:30:16 +0300 Subject: [PATCH 11/18] use default_branch variable --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1e163f9f2a..28723b117a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,6 +6,7 @@ pipeline { environment { GIT_NAME = "geonetwork-eea" registry = "eeacms/eea-geonetwork" + default_branch = "eea-4.2.0" } stages { @@ -16,7 +17,7 @@ pipeline { } steps { script{ - if (env.BRANCH_NAME == 'eea-4.2.0') { + if (env.BRANCH_NAME == env.default_branch ) { tagName = GIT_COMMIT.take(8) } else { tagName = "$BRANCH_NAME" From ca58432b1c788e8fba4712b247926f66e6b07ce5 Mon Sep 17 00:00:00 2001 From: valentinab25 <30239069+valentinab25@users.noreply.github.com> Date: Thu, 19 Jan 2023 11:13:09 +0200 Subject: [PATCH 12/18] Jenkins: keep only CulpritsRecipientProvider --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 28723b117a..3bb1474526 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -48,7 +48,7 @@ pipeline { def details = """

${env.JOB_NAME} - Build #${env.BUILD_NUMBER} - ${status}

Check console output at ${env.JOB_BASE_NAME} - #${env.BUILD_NUMBER}

""" - emailext (subject: summary, body: details, attachLog: true, compressLog: true, recipientProviders: [[$class: 'DevelopersRecipientProvider'], [$class: 'CulpritsRecipientProvider']]) + emailext (subject: summary, body: details, attachLog: true, compressLog: true, recipientProviders: [ [$class: 'CulpritsRecipientProvider']]) } } } From f7f5d624127d0ae903bc2fe189e99a3172b869f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Rodri=CC=81guez?= Date: Thu, 28 Sep 2023 15:55:26 +0200 Subject: [PATCH 13/18] Aling Docker image with official GN image --- Jenkinsfile | 39 ++--- build-in-docker/Dockerfile | 54 ++++--- build-in-docker/build_last_image.sh | 7 +- build-in-docker/docker-entrypoint.sh | 151 ++---------------- .../jetty/geonetwork_context_template.xml | 17 ++ pom.xml | 1 - web/pom.xml | 30 ---- 7 files changed, 90 insertions(+), 209 deletions(-) create mode 100644 build-in-docker/jetty/geonetwork_context_template.xml diff --git a/Jenkinsfile b/Jenkinsfile index 3bb1474526..8f96a40622 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ pipeline { agent { - node { label "docker-host" } + node { label "docker-host" } } environment { @@ -10,31 +10,32 @@ pipeline { } stages { - + stage ('Docker build and push') { when { - environment name: 'CHANGE_ID', value: '' + environment name: 'CHANGE_ID', value: '' } + steps { script{ - if (env.BRANCH_NAME == env.default_branch ) { - tagName = GIT_COMMIT.take(8) - } else { - tagName = "$BRANCH_NAME" - } - try { - dockerImage = docker.build("$registry:$tagName", "--no-cache ./build-in-docker/") - docker.withRegistry( '', 'eeajenkins' ) { - dockerImage.push() - } - } - finally { - sh "docker rmi $registry:$tagName" - } + if (env.BRANCH_NAME == env.default_branch ) { + tagName = GIT_COMMIT.take(8) + } else { + tagName = "$BRANCH_NAME" + } + try { + dockerImage = docker.build("$registry:$tagName", "--no-cache --build-arg COMMIT_OR_BRANCH=$tagName ./build-in-docker/") + docker.withRegistry( '', 'eeajenkins' ) { + dockerImage.push() + } + } + finally { + sh "docker rmi $registry:$tagName" } - } } - + } + } + } post { diff --git a/build-in-docker/Dockerfile b/build-in-docker/Dockerfile index 35d28ea6d3..4c8aa94fd3 100644 --- a/build-in-docker/Dockerfile +++ b/build-in-docker/Dockerfile @@ -1,39 +1,51 @@ -FROM jetty:9-jdk8-eclipse-temurin as build +FROM maven:3-eclipse-temurin-8 AS build + +ARG COMMIT_OR_BRANCH=eea-4.2.0 -USER root RUN apt-get -y update && \ - apt-get install -y maven \ - git \ - zip + apt-get install -y --no-install-recommends \ + unzip + + +RUN git clone --recursive https://github.com/eea/geonetwork-eea.git /tmp/geonetwork-eea +WORKDIR /tmp/geonetwork-eea +RUN git checkout $COMMIT_OR_BRANCH +RUN git submodule update --init --recursive +RUN mvn install -DskipTests -Penv-catalogue +RUN unzip /tmp/geonetwork-eea/web/target/catalogue.war -d /tmp/geonetwork -USER jetty -RUN git clone --recursive https://github.com/eea/geonetwork-eea.git -RUN cd geonetwork-eea && \ - mvn clean install -DskipTests -RUN cd geonetwork-eea/web && \ - mvn package -DskipTests -Penv-catalogue +FROM jetty:9-jdk8-eclipse-temurin AS final +LABEL maintainer="michimau " -RUN unzip /var/lib/jetty/geonetwork-eea/web/target/catalogue.war -d /var/lib/jetty/webapps/catalogue +ENV DATA_DIR /catalogue-data +ENV WEBAPP_CONTEXT_PATH /geonetwork +ENV GN_CONFIG_PROPERTIES -Dgeonetwork.dir=${DATA_DIR} \ + -Dgeonetwork.formatter.dir=${DATA_DIR}/data/formatter \ + -Dgeonetwork.schema.dir=/opt/geonetwork/WEB-INF/data/config/schema_plugins \ + -Dgeonetwork.indexConfig.dir=/opt/geonetwork/WEB-INF/data/config/index -FROM jetty:9-jdk8-eclipse-temurin -MAINTAINER michimau +ENV JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -Djava.awt.headless=true \ + -Xms512M -Xss512M -Xmx2G -XX:+UseConcMarkSweepGC + USER root RUN rm -rf /var/lib/jetty/webapps/* && \ chown jetty:jetty /var/lib/jetty/webapps && \ - mkdir -p /catalogue-data && \ - chown -R jetty:jetty /catalogue-data - -COPY --from=build /var/lib/jetty/webapps/catalogue /var/lib/jetty/webapps/catalogue - -COPY docker-entrypoint.sh / + mkdir -p ${DATA_DIR} && \ + chown -R jetty:jetty ${DATA_DIR} USER jetty +COPY jetty/geonetwork_context_template.xml /usr/local/share/geonetwork/geonetwork_context_template.xml +RUN java -jar /usr/local/jetty/start.jar --create-startd --add-module=http-forwarded +COPY docker-entrypoint.sh /geonetwork-entrypoint.sh + +COPY --chown=jetty:jetty --from=build /tmp/geonetwork /opt/geonetwork -ENTRYPOINT ["/docker-entrypoint.sh"] +ENTRYPOINT ["/geonetwork-entrypoint.sh"] +CMD ["java","-jar","/usr/local/jetty/start.jar"] diff --git a/build-in-docker/build_last_image.sh b/build-in-docker/build_last_image.sh index e7c082b1c8..302ed1a55d 100755 --- a/build-in-docker/build_last_image.sh +++ b/build-in-docker/build_last_image.sh @@ -1,3 +1,8 @@ #!/bin/bash -docker build -t eeacms/geonetwork-eea:$(git ls-remote https://github.com/eea/geonetwork-eea.git HEAD | cut -f1) . +GIT_COMMIT=$(git ls-remote https://github.com/eea/geonetwork-eea.git HEAD | cut -f1) +echo "Building https://github.com/eea/geonetwork-eea.git at $GIT_COMMIT" +docker build \ + -t eeacms/geonetwork-eea:${GIT_COMMIT} \ + --build-arg COMMIT_OR_BRANCH=${GIT_COMMIT} \ + . diff --git a/build-in-docker/docker-entrypoint.sh b/build-in-docker/docker-entrypoint.sh index 63225ca2ba..69c125700e 100755 --- a/build-in-docker/docker-entrypoint.sh +++ b/build-in-docker/docker-entrypoint.sh @@ -1,147 +1,24 @@ -#!/bin/sh - +#!/bin/bash set -e -mkdir -p /catalogue-data/htmlcache -mkdir -p /catalogue-data/lucene -mkdir -p /catalogue-data/logs - -if [ "${ES_HOST}" != "localhost" ]; then - sed -i "s#http://localhost:9200#${ES_PROTOCOL:="http"}://${ES_HOST}:${ES_PORT:="9200"}#g" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/web.xml" ; - sed -i "s#es.host=localhost#es.host=${ES_HOST}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; -fi; - -if [ -n "${ES_PROTOCOL}" ] && [ "${ES_PROTOCOL}" != "http" ] ; then - sed -i "s#es.protocol=http#es.protocol=${ES_PROTOCOL}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; -fi - -if [ -n "${ES_PORT}" ] && [ "$ES_PORT" != "9200" ] ; then - sed -i "s#es.port=9200#es.port=${ES_PORT}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; -fi - -if [ -n "${ES_INDEX_RECORDS}" ] && [ "$ES_INDEX_RECORDS" != "gn-records" ] ; then - sed -i "s#es.index.records=gn-records#es.index.records=${ES_INDEX_RECORDS}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; -fi - -if [ "${ES_USERNAME}" != "" ] ; then - sed -i "s#es.username=#es.username=${ES_USERNAME}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; -fi - -if [ "${ES_PASSWORD}" != "" ] ; then - sed -i "s#es.password=#es.password=${ES_PASSWORD}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; -fi - -if [ -n "${KB_URL}" ] && [ "$KB_URL" != "http://localhost:5601" ]; then - sed -i "s#kb.url=http://localhost:5601#kb.url=${KB_URL}#" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/config.properties" ; - sed -i "s#http://localhost:5601#${KB_URL}#g" "${JETTY_BASE}/webapps/$CATALOGUE/WEB-INF/web.xml" ; -fi - -if [ "$1" = jetty.sh ]; then - if ! command -v bash >/dev/null 2>&1 ; then - cat >&2 <<- 'EOWARN' - ******************************************************************** - ERROR: bash not found. Use of jetty.sh requires bash. - ******************************************************************** - EOWARN - exit 1 - fi - cat >&2 <<- 'EOWARN' - ******************************************************************** - WARNING: Use of jetty.sh from this image is deprecated and may - be removed at some point in the future. +export JAVA_OPTIONS="${JAVA_OPTS} ${GN_CONFIG_PROPERTIES}" - See the documentation for guidance on extending this image: - https://github.com/docker-library/docs/tree/master/jetty - ******************************************************************** - EOWARN -fi +GN_BASE_DIR=/opt/geonetwork if ! command -v -- "$1" >/dev/null 2>&1 ; then set -- java -jar "$JETTY_HOME/start.jar" "$@" fi -: ${TMPDIR:=/tmp/jetty} -[ -d "$TMPDIR" ] || mkdir -p $TMPDIR 2>/dev/null - -: ${JETTY_START:=$JETTY_BASE/jetty.start} - -case "$JAVA_OPTIONS" in - *-Djava.io.tmpdir=*) ;; - *) JAVA_OPTIONS="-Djava.io.tmpdir=$TMPDIR $JAVA_OPTIONS" ;; -esac +if [[ "$1" = jetty.sh ]] || [[ $(expr "$*" : 'java .*/start\.jar.*$') != 0 ]]; then + # Customize context path + if [ ! -f "${JETTY_BASE}/webapps/geonetwork.xml" ]; then + echo "Using ${WEBAPP_CONTEXT_PATH} for deploying the application" + cp /usr/local/share/geonetwork/geonetwork_context_template.xml "${JETTY_BASE}/webapps/geonetwork.xml" + sed -i "s#GEONETWORK_CONTEXT_PATH#${WEBAPP_CONTEXT_PATH}#" "${JETTY_BASE}/webapps/geonetwork.xml" + fi -if expr "$*" : 'java .*/start\.jar.*$' >/dev/null ; then - # this is a command to run jetty - - # check if it is a terminating command - for A in "$@" ; do - case $A in - --add-to-start* |\ - --create-files |\ - --create-startd |\ - --download |\ - --dry-run |\ - --exec-print |\ - --help |\ - --info |\ - --list-all-modules |\ - --list-classpath |\ - --list-config |\ - --list-modules* |\ - --stop |\ - --update-ini |\ - --version |\ - -v )\ - # It is a terminating command, so exec directly - JAVA="$1" - shift - exec $JAVA $JAVA_OPTIONS "$@" - esac - done - - if [ $(whoami) != "jetty" ]; then - cat >&2 <<- EOWARN - ******************************************************************** - WARNING: User is $(whoami) - The user should be (re)set to 'jetty' in the Dockerfile - ******************************************************************** - EOWARN - fi - - if [ -f $JETTY_START ] ; then - if [ $JETTY_BASE/start.d -nt $JETTY_START ] ; then - cat >&2 <<- EOWARN - ******************************************************************** - WARNING: The $JETTY_BASE/start.d directory has been modified since - the $JETTY_START files was generated. Either delete - the $JETTY_START file or re-run - /generate-jetty.start.sh - from a Dockerfile - ******************************************************************** - EOWARN - fi - echo $(date +'%Y-%m-%d %H:%M:%S.000'):INFO:docker-entrypoint:jetty start from $JETTY_START - set -- $(cat $JETTY_START) - else - # Do a jetty dry run to set the final command - JAVA="$1" - shift - $JAVA $JAVA_OPTIONS "$@" --dry-run > $JETTY_START - if [ $(egrep -v '\\$' $JETTY_START | wc -l ) -gt 1 ] ; then - # command was more than a dry-run - cat $JETTY_START \ - | awk '/\\$/ { printf "%s", substr($0, 1, length($0)-1); next } 1' \ - | egrep -v '[^ ]*java .* org\.eclipse\.jetty\.xml\.XmlConfiguration ' - exit - fi - set -- $(sed -e 's/ -Djava.io.tmpdir=[^ ]*//g' -e 's/\\$//' $JETTY_START) - fi + # Delegate on base image entrypoint to start jetty + exec /docker-entrypoint.sh "$@" +else + exec "$@" fi - -if [ "${1##*/}" = java -a -n "$JAVA_OPTIONS" ] ; then - JAVA="$1" - shift - set -- "$JAVA" $JAVA_OPTIONS "$@" -fi - -exec "$@" diff --git a/build-in-docker/jetty/geonetwork_context_template.xml b/build-in-docker/jetty/geonetwork_context_template.xml new file mode 100644 index 0000000000..d667206133 --- /dev/null +++ b/build-in-docker/jetty/geonetwork_context_template.xml @@ -0,0 +1,17 @@ + + + + + GEONETWORK_CONTEXT_PATH + /opt/geonetwork + + -javax.mail. + + + javax.mail. + + + org.eclipse.jetty.server.webapp.WebInfIncludeJarPattern + nomatches + + diff --git a/pom.xml b/pom.xml index c136a23f4a..00e4ef0e91 100644 --- a/pom.xml +++ b/pom.xml @@ -1305,7 +1305,6 @@ healthmonitor services wro4j - web inspire-atom doi es diff --git a/web/pom.xml b/web/pom.xml index d48ece0c65..7f917ea303 100644 --- a/web/pom.xml +++ b/web/pom.xml @@ -1229,8 +1229,6 @@ - geonetwork - config-security prod @@ -1244,9 +1242,6 @@ prod - config-security - geonetwork - config-security-noldap.xml @@ -1259,34 +1254,9 @@ dev - ${geonetwork.webapp.dir}/WEB-INF/data - config-security - geonetwork - config-security-noldap.xml true - - env-catalogue - - - env - prod - - - - prod - config-securitycatalogue - catalogue - /catalogue-data/logs/catalogue.log - file:///catalogue-data - - config-security-noldap.xml - xml/schemas/**,WEB-INF/web*.xml - Editor - SDI_EDITING_DEFAULT - - env-inspire From 5173e6d684befc137dc38e8d8ad745a4f49df507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Rodri=CC=81guez?= Date: Thu, 28 Sep 2023 16:02:51 +0200 Subject: [PATCH 14/18] !fixup Fix WAR name --- build-in-docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-in-docker/Dockerfile b/build-in-docker/Dockerfile index 4c8aa94fd3..97b64faf54 100644 --- a/build-in-docker/Dockerfile +++ b/build-in-docker/Dockerfile @@ -12,7 +12,7 @@ WORKDIR /tmp/geonetwork-eea RUN git checkout $COMMIT_OR_BRANCH RUN git submodule update --init --recursive RUN mvn install -DskipTests -Penv-catalogue -RUN unzip /tmp/geonetwork-eea/web/target/catalogue.war -d /tmp/geonetwork +RUN unzip /tmp/geonetwork-eea/web/target/geonetwork.war -d /tmp/geonetwork FROM jetty:9-jdk8-eclipse-temurin AS final From 53ab17fde44cdb3d164f0dcb049c6fa35546aa96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Rodri=CC=81guez?= Date: Thu, 28 Sep 2023 16:15:49 +0200 Subject: [PATCH 15/18] !fixup env-catalogue profile has been removed --- build-in-docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-in-docker/Dockerfile b/build-in-docker/Dockerfile index 97b64faf54..c936588730 100644 --- a/build-in-docker/Dockerfile +++ b/build-in-docker/Dockerfile @@ -11,7 +11,7 @@ RUN git clone --recursive https://github.com/eea/geonetwork-eea.git /tmp/geonetw WORKDIR /tmp/geonetwork-eea RUN git checkout $COMMIT_OR_BRANCH RUN git submodule update --init --recursive -RUN mvn install -DskipTests -Penv-catalogue +RUN mvn install -DskipTests RUN unzip /tmp/geonetwork-eea/web/target/geonetwork.war -d /tmp/geonetwork From 1bd318df379eb34c3bc0aa968ff2097c3961dafd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Rodri=CC=81guez?= Date: Thu, 28 Sep 2023 17:52:12 +0200 Subject: [PATCH 16/18] Pull latest base image in Jenkins --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8f96a40622..19de07b20b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,7 +24,7 @@ pipeline { tagName = "$BRANCH_NAME" } try { - dockerImage = docker.build("$registry:$tagName", "--no-cache --build-arg COMMIT_OR_BRANCH=$tagName ./build-in-docker/") + dockerImage = docker.build("$registry:$tagName", "--pull --no-cache --build-arg COMMIT_OR_BRANCH=$tagName ./build-in-docker/") docker.withRegistry( '', 'eeajenkins' ) { dockerImage.push() } From 7896ac930ccb8ef344e59a2e58d9921acd651c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Rodri=CC=81guez?= Date: Thu, 28 Sep 2023 18:02:28 +0200 Subject: [PATCH 17/18] Use Maven in Ubuntu Focal for building --- build-in-docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-in-docker/Dockerfile b/build-in-docker/Dockerfile index c936588730..ea94d51ed2 100644 --- a/build-in-docker/Dockerfile +++ b/build-in-docker/Dockerfile @@ -1,4 +1,4 @@ -FROM maven:3-eclipse-temurin-8 AS build +FROM maven:3-eclipse-temurin-8-focal AS build ARG COMMIT_OR_BRANCH=eea-4.2.0 @@ -11,7 +11,7 @@ RUN git clone --recursive https://github.com/eea/geonetwork-eea.git /tmp/geonetw WORKDIR /tmp/geonetwork-eea RUN git checkout $COMMIT_OR_BRANCH RUN git submodule update --init --recursive -RUN mvn install -DskipTests +RUN --mount=type=cache,target=/root/.m2/repository mvn -B install -DskipTests RUN unzip /tmp/geonetwork-eea/web/target/geonetwork.war -d /tmp/geonetwork From 3a199905d2d5a954079f192dd971529cad9cb772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Rodri=CC=81guez?= Date: Thu, 28 Sep 2023 18:07:06 +0200 Subject: [PATCH 18/18] Remove unsupported mount flag too new for the docker engine used --- build-in-docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-in-docker/Dockerfile b/build-in-docker/Dockerfile index ea94d51ed2..7dd098706c 100644 --- a/build-in-docker/Dockerfile +++ b/build-in-docker/Dockerfile @@ -11,7 +11,7 @@ RUN git clone --recursive https://github.com/eea/geonetwork-eea.git /tmp/geonetw WORKDIR /tmp/geonetwork-eea RUN git checkout $COMMIT_OR_BRANCH RUN git submodule update --init --recursive -RUN --mount=type=cache,target=/root/.m2/repository mvn -B install -DskipTests +RUN mvn -B install -DskipTests RUN unzip /tmp/geonetwork-eea/web/target/geonetwork.war -d /tmp/geonetwork