EEA / Helper script.

eea · Mar 21, 2024 · c12306d · c12306d
1 parent d10fe2f
commit c12306d
Show file tree

Hide file tree

Showing 9 changed files with 523 additions and 2 deletions.
diff --git a/eea/collection-builder/updateseries.sh b/eea/collection-builder/updateseries.sh
@@ -0,0 +1,78 @@
+#SERVER=http://localhost:8080/geonetwork
+#SERVER=https://sdi.eea.europa.eu/catalogue
+SERVER=https://galliwasp.eea.europa.eu/catalogue
+CATALOGUSER=
+CATALOGPASS=
+AUTH="-u $CATALOGUSER:$CATALOGPASS"
+#AUTH=""
+
+
+type=series
+from=0
+size=1000
+
+rm results.json
+rm -f /tmp/cookie;
+
+curl -s -c /tmp/cookie -o /dev/null \
+  -X GET \
+  --user $CATALOGUSER:$CATALOGPASS \
+  -H "Accept: application/json" \
+  "$SERVER/srv/api/me";
+
+export TOKEN=`grep XSRF-TOKEN /tmp/cookie | cut -f 7`;
+export JSESSIONID=`grep JSESSIONID /tmp/cookie | cut -f 7`;
+
+echo "Using session with $TOKEN and id $JSESSIONID"
+
+curl $AUTH "$SERVER/srv/api/search/records/_search" \
+    -X 'POST' \
+    -H 'Accept: application/json, text/plain, */*' \
+    -H 'Content-Type: application/json;charset=UTF-8' \
+    --data-raw "{\"query\":{\"query_string\":{\"query\": \"+isHarvested:false +resourceType: $type\"}},\"from\":$from, \"size\":$size, \"_source\": {\"include\": [\"resourceTitleObject.default\"]}, \"sort\": [{\"resourceTitleObject.default.keyword\": \"asc\"}]}" \
+    -H "X-XSRF-TOKEN: $TOKEN" -H "Cookie: XSRF-TOKEN=$TOKEN; JSESSIONID=$JSESSIONID" \
+    --compressed \
+    -o results.json
+
+for hit in $(jq -r '.hits.hits[] | @base64' results.json); do
+   _jq() {
+     echo "${hit}" | base64 --decode | jq -r "${1}"
+    }
+
+  title=$(_jq '._source.resourceTitleObject.default')
+  uuid=$(_jq '._id')
+  echo "__________"
+  echo "### $uuid"
+
+  curl $AUTH "$SERVER/srv/api/records/$uuid/processes/collection-updater" \
+    -X 'POST' \
+    -H 'Accept: application/json, text/plain, */*' \
+    -H "X-XSRF-TOKEN: $TOKEN" \
+    -H "Cookie: XSRF-TOKEN=$TOKEN; JSESSIONID=$JSESSIONID" \
+    --compressed
+
+  echo "Clean selection:"
+  curl $AUTH "$SERVER/srv/api/selections/s101" \
+    -X 'DELETE' \
+    -H 'Accept: application/json, text/javascript, */*; q=0.01' \
+    -H "X-XSRF-TOKEN: $TOKEN" \
+    -H "Cookie: XSRF-TOKEN=$TOKEN; JSESSIONID=$JSESSIONID" \
+    --compressed
+
+  curl $AUTH "$SERVER/srv/api/selections/s101?uuid=$uuid" \
+    -X 'PUT' \
+    -H 'Accept: application/json, text/javascript, */*; q=0.01' \
+    -H "X-XSRF-TOKEN: $TOKEN" \
+    -H "Cookie: XSRF-TOKEN=$TOKEN; JSESSIONID=$JSESSIONID" \
+    --compressed
+
+  curl $AUTH "$SERVER/srv/api/records/batchediting?bucket=s101" \
+    -X 'PUT' \
+    -H 'Accept: application/json, text/plain, */*' \
+    -H 'Content-Type: application/json;charset=UTF-8' \
+    -H "X-XSRF-TOKEN: $TOKEN" \
+    -H "Cookie: XSRF-TOKEN=$TOKEN; JSESSIONID=$JSESSIONID" \
+    --data-raw "[{\"xpath\":\"/gmd:identificationInfo/*/gmd:citation/*/gmd:identifier[position() > 2]\",\"value\":\"<gn_delete/>\"}]" \
+    --compressed
+done;
+
diff --git a/eea/connect.sh b/eea/connect.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+export SERVER=${1:-https://galliwasp.eea.europa.eu/catalogue}
+#export SERVER=${1:-http://localhost:8080/geonetwork}
+export CATALOGUSER=${2:-default}
+export CATALOGPASS=${3:-default}
+export AUTH="-u $CATALOGUSER:$CATALOGPASS"
+
+echo "Calling API on $SERVER with user $CATALOGUSER."
+
+rm -f /tmp/cookie;
+curl -s -c /tmp/cookie -o /dev/null \
+  $AUTH -X GET  \
+  --user $CATALOGUSER:$CATALOGPASS \
+  -H "Accept: application/json" \
+  "$SERVER/srv/api/me";
+
+export TOKEN=`grep XSRF-TOKEN /tmp/cookie | cut -f 7`;
+export JSESSIONID=`grep JSESSIONID /tmp/cookie | cut -f 7`;
+echo "Connected with user token $TOKEN (session id: $JSESSIONID)."
diff --git a/eea/copernicus/emailupdate.sh b/eea/copernicus/emailupdate.sh
@@ -0,0 +1,53 @@
+#SERVER=http://localhost:8080/geonetwork
+SERVER=https://sdi.eea.europa.eu/catalogue
+CATALOGUSER=
+CATALOGPASS=
+AUTH="-u $CATALOGUSER:$CATALOGPASS"
+
+rm -f /tmp/cookie;
+curl -s -c /tmp/cookie -o /dev/null \
+  -X GET -v \
+  --user $CATALOGUSER:$CATALOGPASS \
+  -H "Accept: application/json" \
+  "$SERVER/srv/api/me";
+
+export TOKEN=`grep XSRF-TOKEN /tmp/cookie | cut -f 7`;
+export JSESSIONID=`grep JSESSIONID /tmp/cookie | cut -f 7`;
+
+echo "Using session with $TOKEN and id $JSESSIONID"
+
+while IFS=";" read -r uuid; do
+  echo ""
+  echo "____________________________________"
+  echo "$uuid"
+
+  curl $AUTH "$SERVER/srv/api/selections/s101" \
+    -X 'DELETE' \
+    -H 'Accept: application/json, text/javascript, */*; q=0.01' \
+    -H "X-XSRF-TOKEN: $TOKEN" \
+    -H "Cookie: XSRF-TOKEN=$TOKEN; JSESSIONID=$JSESSIONID" \
+    --compressed
+
+  curl $AUTH "$SERVER/srv/api/selections/s101?uuid=$uuid" \
+    -X 'PUT' \
+    -H 'Accept: application/json, text/javascript, */*; q=0.01' \
+    -H "X-XSRF-TOKEN: $TOKEN" \
+    -H "Cookie: XSRF-TOKEN=$TOKEN; JSESSIONID=$JSESSIONID" \
+    --compressed
+
+  read -r -d '' batchEditConfig << EOF
+    {
+      "xpath":".//gmd:pointOfContact[gmd:CI_ResponsibleParty/gmd:contactInfo/*/gmd:address/*/gmd:electronicMailAddress/gco:CharacterString = '[email protected]']/@gco:nilReason[. = 'withheld']",
+      "value":"<gn_delete/>"
+    }
+EOF
+
+  curl -v $AUTH "$SERVER/srv/api/records/batchediting?bucket=s101" \
+    -X 'PUT' \
+    -H 'Accept: application/json, text/plain, */*' \
+    -H 'Content-Type: application/json;charset=UTF-8' \
+    -H "X-XSRF-TOKEN: $TOKEN" \
+    -H "Cookie: XSRF-TOKEN=$TOKEN; JSESSIONID=$JSESSIONID" \
+    --data-raw "[${batchEditConfig//\\n/}]" \
+    --compressed
+done < email.csv
diff --git a/eea/copernicus/updateobsolete.sh b/eea/copernicus/updateobsolete.sh
@@ -0,0 +1,120 @@
+#SERVER=http://localhost:8080/geonetwork
+SERVER=https://galliwasp.eea.europa.eu/catalogue
+CATALOGUSER=
+CATALOGPASS=
+AUTH="-u $CATALOGUSER:$CATALOGPASS"
+
+from=0
+size=1000
+echo $AUTH
+
+rm results.json
+rm -f /tmp/cookie;
+
+curl -s -c /tmp/cookie -o /dev/null \
+  -X GET -v \
+  --user $CATALOGUSER:$CATALOGPASS \
+  -H "Accept: application/json" \
+  "$SERVER/srv/api/me";
+
+export TOKEN=`grep XSRF-TOKEN /tmp/cookie | cut -f 7`;
+export JSESSIONID=`grep JSESSIONID /tmp/cookie | cut -f 7`;
+export TOKEN="9ab46ed8-b98c-4a96-97b1-33ae87b9d6d6";
+export JSESSIONID="node0zc7h7tpzn0ej5jaasp7by4673000.node0";
+
+echo "Using session with $TOKEN and id $JSESSIONID"
+
+curl $AUTH "$SERVER/srv/api/search/records/_search" \
+    -X 'POST' \
+    -H 'Accept: application/json, text/plain, */*' \
+    -H 'Content-Type: application/json;charset=UTF-8' \
+    --data-raw "{\"query\":{\"query_string\":{\"query\": \"+isHarvested:false  +linkUrl:/.*land.copernicus.eu.*/ +cl_status.key:obsolete\"}},\"from\":$from, \"size\":$size, \"_source\": {\"include\": [\"resourceTitleObject.default\"]}, \"sort\": [{\"resourceTitleObject.default.keyword\": \"asc\"}]}" \
+    -H "X-XSRF-TOKEN: $TOKEN" -H "Cookie: XSRF-TOKEN=$TOKEN; JSESSIONID=$JSESSIONID" \
+    --compressed \
+    -o results.json
+
+for hit in $(jq -r '.hits.hits[] | @base64' results.json); do
+   _jq() {
+     echo "${hit}" | base64 --decode | jq -r "${1}"
+    }
+
+  title=$(_jq '._source.resourceTitleObject.default')
+  uuid=$(_jq '._id')
+  echo "\n__________"
+  echo "### $uuid - $title"
+
+  echo "Clean selection:"
+  curl $AUTH "$SERVER/srv/api/selections/s101" \
+    -X 'DELETE' \
+    -H 'Accept: application/json, text/javascript, */*; q=0.01' \
+    -H "X-XSRF-TOKEN: $TOKEN" \
+    -H "Cookie: XSRF-TOKEN=$TOKEN; JSESSIONID=$JSESSIONID" \
+    --compressed
+
+  curl $AUTH "$SERVER/srv/api/selections/s101?uuid=$uuid" \
+    -X 'PUT' \
+    -H 'Accept: application/json, text/javascript, */*; q=0.01' \
+    -H "X-XSRF-TOKEN: $TOKEN" \
+    -H "Cookie: XSRF-TOKEN=$TOKEN; JSESSIONID=$JSESSIONID" \
+    --compressed
+
+
+  read -r -d '' xmlSnippet << EOF
+  <gn_add><gmd:distributor xmlns:gmd="http://www.isotc211.org/2005/gmd"
+                                  xmlns:gco="http://www.isotc211.org/2005/gco">
+                             <gmd:MD_Distributor>
+                                <gmd:distributorContact>
+                                   <gmd:CI_ResponsibleParty>
+                                      <gmd:organisationName>
+                                         <gco:CharacterString>Copernicus Land Monitoring Service</gco:CharacterString>
+                                      </gmd:organisationName>
+                                      <gmd:contactInfo>
+                                         <gmd:CI_Contact>
+                                            <gmd:address>
+                                               <gmd:CI_Address>
+                                                  <gmd:electronicMailAddress>
+                                                     <gco:CharacterString>[email protected]</gco:CharacterString>
+                                                  </gmd:electronicMailAddress>
+                                               </gmd:CI_Address>
+                                            </gmd:address>
+                                         </gmd:CI_Contact>
+                                      </gmd:contactInfo>
+                                      <gmd:role>
+                                         <gmd:CI_RoleCode codeList="http://standards.iso.org/iso/19139/resources/gmxCodelists.xml#CI_RoleCode"
+                                                          codeListValue="distributor"/>
+                                      </gmd:role>
+                                   </gmd:CI_ResponsibleParty>
+                                </gmd:distributorContact>
+                                <gmd:distributionOrderProcess>
+                                   <gmd:MD_StandardOrderProcess>
+                                      <gmd:orderingInstructions>
+                                         <gco:CharacterString>This dataset is no longer accessible from the website of the Copernicus Land Monitoring Service.
+                 You can request access to this dataset by contacting the service desk of the Copernicus Land Monitoring Service at [email protected].</gco:CharacterString>
+                                      </gmd:orderingInstructions>
+                                   </gmd:MD_StandardOrderProcess>
+                                </gmd:distributionOrderProcess>
+                             </gmd:MD_Distributor>
+                          </gmd:distributor></gn_add>
+EOF
+  xmlSnippetWithNoNewLine="${xmlSnippet//$'\n'/}"
+  read -r -d '' batchEditConfig << EOF
+    {
+      "xpath":"/gmd:distributionInfo/*/gmd:transferOptions/*/gmd:onLine[starts-with(*/gmd:linkage/gmd:URL, 'https://land.copernicus.eu/')]",
+      "value":"<gn_delete/>"
+    },
+    {
+      "xpath":"/gmd:distributionInfo[1]/*",
+      "value":"${xmlSnippetWithNoNewLine//\"/\\\"}"
+    }
+EOF
+
+  curl -v $AUTH "$SERVER/srv/api/records/batchediting?bucket=s101" \
+    -X 'PUT' \
+    -H 'Accept: application/json, text/plain, */*' \
+    -H 'Content-Type: application/json;charset=UTF-8' \
+    -H "X-XSRF-TOKEN: $TOKEN" \
+    -H "Cookie: XSRF-TOKEN=$TOKEN; JSESSIONID=$JSESSIONID" \
+    --data-raw "[${batchEditConfig//\\n/}]" \
+    --compressed
+done;
+
diff --git a/eea/featurecatalogues/extractDataTable.sh b/eea/featurecatalogues/extractDataTable.sh
@@ -0,0 +1,62 @@
+
+SERVER=http://localhost:8080/geonetwork
+CATALOGUSER=
+CATALOGPASS=
+AUTH=""
+
+rm results.json
+rm -f /tmp/cookie;
+
+curl -s -c /tmp/cookie -o /dev/null \
+  -X GET  \
+  --user $CATALOGUSER:$CATALOGPASS \
+  -H "Accept: application/json" \
+  "$SERVER/srv/api/me";
+
+export TOKEN=`grep XSRF-TOKEN /tmp/cookie | cut -f 7`;
+export JSESSIONID=`grep JSESSIONID /tmp/cookie | cut -f 7`;
+
+echo "Using session with $TOKEN and id $JSESSIONID"
+
+GROUP=1069874
+
+rm fc.csv
+for f in $(ls ../dataandmaps/data/*.rdf); do
+  i=0
+  echo $f
+  for parts in $(xmllint --xpath "//*[local-name() = 'hasPart']/@*[local-name() = 'resource']" $f | grep -v ">" | cut -f 2 -d "=" | tr -d \"); do
+    partFile=$(basename $f .rdf)-$i.rdf
+    # curl -s "$parts/@@rdf" --output "fc2/$partFile"
+
+    tableTitle=$(xmllint --xpath "//*[local-name() = 'DataTable']/*[local-name() = 'title']" fc2/$partFile)
+    tableDesc=$(xmllint --xpath "//*[local-name() = 'DataTable']/*[local-name() = 'tableDefinition']" fc2/$partFile)
+
+    tableDefinition=$(xmllint --xpath "//*[local-name() = 'tableDefinition']" fc2/$partFile)
+    tableCmsId=$(xmllint --xpath "//*[local-name() = 'DataTable']/@*[local-name() = 'about']" fc2/$partFile)
+    title=$(xmllint --xpath "//*[local-name() = 'DataTable']/*[local-name() = 'title']" fc2/$partFile)
+    abs=$(xmllint --xpath "//*[local-name() = 'DataTable']/*[local-name() = 'description']" fc2/$partFile)
+
+    echo "Feature catalogue: $tableTitle"
+    echo "$tableDefinition"
+
+
+    if [[ ! -z "$tableDefinition" ]]; then
+      fcfile="fc2/$partFile.xml"
+      echo "<fc><uuid>$uuid</uuid><url>$url</url><fcurl>$parts</fcurl>$title$abs$tableDefinition</fc>" > $fcfile
+
+      sed -i 's/<\/dcterms:title>/ - feature catalogue<\/title>/g' $fcfile
+      sed -i 's/dcterms://g' $fcfile
+      sed -i 's/datatable://g' $fcfile
+
+      echo "Importing CMS record from file $fcfile"
+      curl -s "$SERVER/srv/api/records" -X POST \
+        -H 'Accept: application/json, text/javascript, */*; q=0.01' \
+        -H "X-XSRF-TOKEN: $TOKEN" \
+        -H 'Content-Type: multipart/form-data' \
+        -H "Cookie: XSRF-TOKEN=$TOKEN; JSESSIONID=$JSESSIONID" \
+        -F "metadataType=METADATA" -F "uuidProcessing=GENERATEUUID" \
+        -F "transformWith=EEAFCRDF-to-ISO19115-3" -F "group=$GROUP" -F "file=@$fcfile" > $fcfile.json
+    fi
+    ((i=i+1))
+  done
+done
diff --git a/eea/managementplan/convert.sh b/eea/managementplan/convert.sh
@@ -1,3 +1,3 @@
-sed -i.ini 's/ \& / \&amp; /g'  management_plan2023.xml
+sed -i.ini 's/ \& / \&amp; /g'  management_plan2024.xml
 #../../web/src/main/webapp/WEB-INF/data/config/codelist/local/thesauri/theme/
-xsltproc -o eea-mp.rdf eeamp2skos.xsl management_plan2023.xml
+xsltproc -o eea-mp.rdf eeamp2skos.xsl management_plan2024.xml
diff --git a/eea/tabulardatadownload.sh b/eea/tabulardatadownload.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+TARGETDIRECTORY=/home/lubuntu/docker_work/tabular2sdi/tabular/
+
+curl -q 'https://semantic.eea.europa.eu/sparql?selectedBookmarkName=&query=PREFIX+DataFileLink%3A+%3Chttp%3A%2F%2Fwww.eea.europa.eu%2Fportal_types%2FDataFileLink%23%3E%0D%0APREFIX+dcterms%3A+%3Chttp%3A%2F%2Fpurl.org%2Fdc%2Fterms%2F%3E%0D%0APREFIX+data%3A+%3Chttp%3A%2F%2Fwww.eea.europa.eu%2Fportal_types%2FData%23%3E%0D%0A%0D%0ASELECT+%0D%0A%3Fdata%0D%0A%3Fid%0D%0A%3FremoteUrl%0D%0Amin%28xsd%3Ainteger%28%3FtemporalCoverage%29%29+AS+%3FminYear%0D%0Amax%28xsd%3Ainteger%28%3FtemporalCoverage%29%29+AS+%3FmaxYear%0D%0A%0D%0AWHERE+%7B%0D%0A++%3FDataFileLink+a+DataFileLink%3ADataFileLink+.%0D%0A++%3FDataFileLink+dcterms%3Atitle+%3Ftitle+.+%0D%0A++%3FDataFileLink+DataFileLink%3AremoteUrl+%3FremoteUrl+.%0D%0A++%3FdataTable+dcterms%3AhasPart+%3FDataFileLink+.%0D%0A++%3Fdata+dcterms%3AhasPart+%3FdataTable+.%0D%0A++%3Fdata+data%3Aid+%3Fid+.%0D%0A++%3Fdata+data%3AtemporalCoverage+%3FtemporalCoverage+.%0D%0A%7D&format=text%2Fcsv&nrOfHits=999&execute=Execute' > sparql.csv
+
+while read p; do
+  ID=$(echo "$p" | cut -f2 -d';')
+  #echo ID= $ID
+  echo p= $p
+
+  VERSION_TMP=$(echo $ID | grep -Eo '[^-]*([0-9]+)$')
+
+  if [[ "${#VERSION_TMP}" -eq 1 ]]; then
+    VERSION='0'$VERSION_TMP
+    BASENAME=${ID::-2}
+  elif [[ "${#VERSION_TMP}" -eq 0 ]]; then
+    VERSION='00'
+    BASENAME=$ID
+  else
+    VERSION='00'
+    BASENAME=$ID
+  fi
+
+  FILEURL=$(echo $p | cut -f3 -d';')
+  echo FILEURL= $FILEURL
+  STARTYEAR=$(echo $p | cut -f4 -d';')
+  ENDYEAR=$(echo $p | cut -f5 -d';' | tr -d '\r')
+
+  if [ $ENDYEAR = $STARTYEAR ]; then
+    DATERANGE=$STARTYEAR
+  else
+    DATERANGE=$STARTYEAR'-'$ENDYEAR
+  fi
+
+  #echo "$p" | cut -f1 -d';'
+  DATADIRECTORY=$(echo 'eea_t_'$BASENAME'_'$DATERANGE'_v'$VERSION'_r00')
+
+  mkdir -p $TARGETDIRECTORY$DATADIRECTORY
+  cd $TARGETDIRECTORY$DATADIRECTORY
+  curl -L -J -O $FILEURL
+  cd -
+done < sparql.csv