Update image versions and configurations

* Updated nginx version * Updated nginx conf for https traffic * Updated postgres version from 9.6 to 13 * Updated elasticsearch version to 2.4.6 * Updated elasticsearch configuration to allow scripting * Added two updated elasticsearch mappings and mounts for them
EUDAT-B2SHARE · May 23, 2023 · a48f933 · a48f933
1 parent b1bd7d4
commit a48f933
Show file tree

Hide file tree

Showing 9 changed files with 227 additions and 7 deletions.
diff --git a/README.md b/README.md
@@ -7,8 +7,14 @@ cp b2share.env .env
 docker-compose up
 ```
 
+## Usage with B2SHARE configloader
+
+With B2SHARE v2.3.0 a new configuration loader has been introduced. Because of this, all environment variables for b2share image has to start with prefix `B2SHARE_`
+
 ## Useful links
 
  * [B2SHARE training module](https://github.com/EUDAT-Training/B2SHARE-Training/tree/master/deploy)
  * [B2SHARE install notes](https://github.com/EUDAT-B2SHARE/b2share/blob/evolution/INSTALL.rst)
 
+Upgrade guide for postgres upgrade [HERE](./upgrade.md)
+
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '2.3'
 services:
     postgres:
-        image: postgres:9.6
+        image: postgres:13
         environment:
             - "POSTGRES_PASSWORD=${B2SHARE_POSTGRESQL_PASSWORD}"
             - "POSTGRES_USER=${B2SHARE_POSTGRESQL_USER}"
@@ -13,10 +13,11 @@ services:
             - "5432"
 
     b2share-base:
-        image: eudatb2share/b2share:2.2.0
+        image: eudatb2share/b2share:2.2.5
         command:
             - /bin/echo
         environment:
+            # If used with b2share configloader, add B2SHARE_ in front of every variable that does not allready have it
             - "B2ACCESS_CONSUMER_KEY=${B2ACCESS_CONSUMER_KEY}"
             - "B2ACCESS_SECRET_KEY=${B2ACCESS_SECRET_KEY}"
             - "USE_STAGING_B2ACCESS=${USE_STAGING_B2ACCESS}"
@@ -35,6 +36,8 @@ services:
             - "B2SHARE_SEARCH_ELASTIC_HOSTS='elasticsearch'"
         volumes:
             - "${B2SHARE_DATADIR}/b2share-data:/usr/var/b2share-instance"
+            - "./elasticsearch/mappings/record-view:/usr/local/lib/python3.6/site-packages/invenio_stats/contrib/record_view/v2"
+            - "./elasticsearch/mappings/file-download:/usr/local/lib/python3.6/site-packages/invenio_stats/contrib/file_download/v2"
 
     b2share:
         extends: b2share-base
@@ -50,7 +53,11 @@ services:
 
     elasticsearch:
         build: elasticsearch
-        image: eudatb2share/elasticsearch:2.2
+        image: eudatb2share/elasticsearch:2.4.6
+        environment:
+            # Set options for elasticsearch memory usage
+            - bootstrap.memory_lock=true
+            - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
         expose:
             - "9200"
             - "9300"

diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile
@@ -1,4 +1,4 @@
-FROM elasticsearch:2.2
+FROM elasticsearch:2.4.6
 EXPOSE 9200 9300
 
 RUN cd /usr/share/elasticsearch && \

diff --git a/elasticsearch/config/elasticsearch.yml b/elasticsearch/config/elasticsearch.yml
@@ -0,0 +1,3 @@
+network.host: 0.0.0.0
+script.inline: on
+script.indexed: on
diff --git a/elasticsearch/mappings/file-download/file-download-v1.json b/elasticsearch/mappings/file-download/file-download-v1.json
@@ -0,0 +1,66 @@
+{
+    "template": "events-stats-file-download-*",
+    "settings": {
+      "index": {
+        "refresh_interval": "5s"
+      }
+    },
+    "mappings": {
+      "_default_": {
+        "dynamic_templates": [
+          {
+            "date_fields": {
+              "match_mapping_type": "date",
+              "mapping": {
+                "type": "date",
+                "format": "strict_date_hour_minute_second"
+              }
+            }
+          }
+        ]
+      },
+      "file-download": {
+        "_source": {
+          "enabled": false
+        },
+        "_all": {
+          "enabled": false
+        },
+        "date_detection": false,
+        "numeric_detection": false,
+        "properties": {
+          "timestamp": {
+            "type": "date",
+            "format": "strict_date_hour_minute_second"
+          },
+          "bucket_id": {
+            "type": "string",
+            "index": "not_analyzed"
+          },
+          "file_id": {
+            "type": "string",
+            "index": "not_analyzed"
+          },
+          "file_key": {
+            "type": "string",
+            "index": "not_analyzed"
+          },
+          "unique_id": {
+            "type": "string",
+            "index": "not_analyzed"
+          },
+          "visitor_id": {
+            "type": "string",
+            "index": "not_analyzed"
+          },
+          "collection": {
+            "type": "string",
+            "index": "not_analyzed"
+          }
+        }
+      }
+    },
+    "aliases": {
+      "events-stats-file-download": {}
+    }
+  }
diff --git a/elasticsearch/mappings/record-view/record-view-v1.json b/elasticsearch/mappings/record-view/record-view-v1.json
@@ -0,0 +1,53 @@
+{
+    "template": "events-stats-record-view-*",
+    "settings": {
+      "index": {
+        "refresh_interval": "5s"
+      }
+    },
+    "mappings": {
+      "record-view": {
+        "_source": {
+          "enabled": false
+        },
+        "_all": {
+          "enabled": false
+        },
+        "date_detection": false,
+        "numeric_detection": false,
+        "properties": {
+          "timestamp": {
+            "type": "date",
+            "format": "strict_date_hour_minute_second"
+          },
+          "record_id": {
+            "type": "string",
+            "index": "not_analyzed"
+          },
+          "pid_type": {
+            "type": "string",
+            "index": "not_analyzed"
+          },
+          "pid_value": {
+            "type": "string",
+            "index": "not_analyzed"
+          },
+          "labels": {
+            "type": "string",
+            "index": "not_analyzed"
+          },
+          "visitor_id": {
+            "type": "string",
+            "index": "not_analyzed"
+          },
+          "community": {
+            "type": "string",
+            "index": "not_analyzed"
+          }
+        }
+      }
+    },
+    "aliases": {
+      "events-stats-record-view": {}
+    }
+  }
diff --git a/nginx/Dockerfile b/nginx/Dockerfile
@@ -1,4 +1,4 @@
-FROM nginx:1.18
+FROM nginx:mainline-alpine
 EXPOSE 80 443
 
 RUN rm /etc/nginx/conf.d/*.conf

diff --git a/nginx/b2share.conf b/nginx/b2share.conf
@@ -2,6 +2,8 @@ server {
         listen 80;
         charset utf-8;
 
+        server_tokens           off;
+
         location /oai2d {
                 proxy_pass http://b2share:5000/api/oai2d;
                 proxy_set_header Host $host;
@@ -23,8 +25,9 @@ server {
         charset utf-8;
         ssl_certificate         /etc/ssl/b2share.crt;
         ssl_certificate_key     /etc/ssl/b2share.key;
-        ssl_protocols           TLSv1 TLSv1.1 TLSv1.2;
-        ssl_ciphers             HIGH:!aNULL:!MD5;
+        ssl_protocols           TLSv1.2 TLSv1.3;
+        ssl_ciphers             !aNULL:!MD5:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+        ssl_prefer_server_ciphers off;
 
         client_body_timeout 600s;
 

diff --git a/upgrade.md b/upgrade.md
@@ -0,0 +1,82 @@
+# Upgrade guide for POSTGRESQL
+
+From 9.6 to 13
+
+**Step 1: Get a database dump**
+
+With a running stack before upgrade:
+
+```bash
+docker-compose exec postgres pg_dump --blobs --clean --dbname="${DBNAME}" --username="${DBUSER}" | gzip > dumpfile.sql.gz
+```
+
+**Step 2: DELETE old db**
+
+With a running stack before upgrade:
+
+```bash
+docker container ls --filter name=<COMPOSE_PROJECT_NAME>_postgres_1 --format {{.ID}}
+```
+
+Variable `COMPOSE_PROJECT_NAME` is define in the .env file of the project. If the env variable is not define, the default value is the name of the folder where the .yml file is built.
+
+```bash
+docker exec -i <CONTAINER ID POSTGRES> psql --user <B2SHARE_POSTGRESQL_USER> postgres -c 'DROP database <B2SHARE_POSTGRESQL_DBNAME>'
+```
+
+**Step 3: Upgrade postgres**
+
+New version of postgres is defined in the `docker-compose.yml` file. Thus, you only need to run:
+
+```bash
+docker-compose up -d postgres
+```
+
+**Step 4: Recreate database**
+
+With a running stack after upgrade:
+
+Get ID of the recreated container (Same as in step 2):
+
+```bash
+docker container ls --filter name=<COMPOSE_PROJECT_NAME>_postgres_1 --format {{.ID}}
+```
+
+```bash
+docker exec -i <CONTAINER ID POSTGRES> psql --user <B2SHARE_POSTGRESQL_USER> postgres -c 'CREATE database <B2SHARE_POSTGRESQL_DBNAME>'
+```
+
+**Step 5: Restore data**
+
+With a running stack after upgrade:
+
+```bash
+# Uncompressed .sql file:
+docker exec -i <CONTAINER ID> psql --user <B2SHARE_POSTGRESQL_USER> <B2SHARE_POSTGRESQL_DBNAME> < <ABSOLUTE_PATH_TO_DB_BACKUP>
+# Gzip compressed .sql.gz file:
+zcat <ABSOLUTE_PATH_TO_DB_BACKUP> | docker exec -i <CONTAINER ID> psql --user <B2SHARE_POSTGRESQL_USER> <B2SHARE_POSTGRESQL_DBNAME>
+```
+
+Now the database should be upgraded to version 13
+
+**If there are issues** it is a good idea to reindex elasticsearch indices:
+
+Get B2SHARE container ID:
+
+```bash
+docker container ls --filter name=<COMPOSE_PROJECT_NAME>_b2share_1 --format {{.ID}}
+```
+
+With container ID in hand, run:
+
+```bash
+docker exec -it <CONTAINER ID B2SHARE> bash
+# These commands executed in B2SHARE container
+b2share index destroy --force --yes-i-know
+b2share index init --force
+b2share index queue init purge
+
+b2share index reindex --yes-i-know
+b2share index run
+
+```