diff --git a/README.md b/README.md index 0aa4b1a..0dc0517 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,14 @@ cp b2share.env .env docker-compose up ``` +## Usage with B2SHARE configloader + +With B2SHARE v2.3.0 a new configuration loader has been introduced. Because of this, all environment variables for b2share image has to start with prefix `B2SHARE_` + ## Useful links * [B2SHARE training module](https://github.com/EUDAT-Training/B2SHARE-Training/tree/master/deploy) * [B2SHARE install notes](https://github.com/EUDAT-B2SHARE/b2share/blob/evolution/INSTALL.rst) +Upgrade guide for postgres upgrade [HERE](./upgrade.md) + diff --git a/docker-compose.yml b/docker-compose.yml index 6ae73ae..045297d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '2.3' services: postgres: - image: postgres:9.6 + image: postgres:13 environment: - "POSTGRES_PASSWORD=${B2SHARE_POSTGRESQL_PASSWORD}" - "POSTGRES_USER=${B2SHARE_POSTGRESQL_USER}" @@ -13,10 +13,11 @@ services: - "5432" b2share-base: - image: eudatb2share/b2share:2.2.0 + image: eudatb2share/b2share:2.2.5 command: - /bin/echo environment: + # If used with b2share configloader, add B2SHARE_ in front of every variable that does not allready have it - "B2ACCESS_CONSUMER_KEY=${B2ACCESS_CONSUMER_KEY}" - "B2ACCESS_SECRET_KEY=${B2ACCESS_SECRET_KEY}" - "USE_STAGING_B2ACCESS=${USE_STAGING_B2ACCESS}" @@ -35,6 +36,8 @@ services: - "B2SHARE_SEARCH_ELASTIC_HOSTS='elasticsearch'" volumes: - "${B2SHARE_DATADIR}/b2share-data:/usr/var/b2share-instance" + - "./elasticsearch/mappings/record-view:/usr/local/lib/python3.6/site-packages/invenio_stats/contrib/record_view/v2" + - "./elasticsearch/mappings/file-download:/usr/local/lib/python3.6/site-packages/invenio_stats/contrib/file_download/v2" b2share: extends: b2share-base @@ -50,7 +53,11 @@ services: elasticsearch: build: elasticsearch - image: eudatb2share/elasticsearch:2.2 + image: eudatb2share/elasticsearch:2.4.6 + environment: + # Set options for elasticsearch memory usage + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" expose: - "9200" - "9300" diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index c9f8372..00620d1 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -1,4 +1,4 @@ -FROM elasticsearch:2.2 +FROM elasticsearch:2.4.6 EXPOSE 9200 9300 RUN cd /usr/share/elasticsearch && \ diff --git a/elasticsearch/config/elasticsearch.yml b/elasticsearch/config/elasticsearch.yml new file mode 100644 index 0000000..a1b6e2e --- /dev/null +++ b/elasticsearch/config/elasticsearch.yml @@ -0,0 +1,3 @@ +network.host: 0.0.0.0 +script.inline: on +script.indexed: on diff --git a/elasticsearch/mappings/file-download/file-download-v1.json b/elasticsearch/mappings/file-download/file-download-v1.json new file mode 100644 index 0000000..e4bfbf3 --- /dev/null +++ b/elasticsearch/mappings/file-download/file-download-v1.json @@ -0,0 +1,66 @@ +{ + "template": "events-stats-file-download-*", + "settings": { + "index": { + "refresh_interval": "5s" + } + }, + "mappings": { + "_default_": { + "dynamic_templates": [ + { + "date_fields": { + "match_mapping_type": "date", + "mapping": { + "type": "date", + "format": "strict_date_hour_minute_second" + } + } + } + ] + }, + "file-download": { + "_source": { + "enabled": false + }, + "_all": { + "enabled": false + }, + "date_detection": false, + "numeric_detection": false, + "properties": { + "timestamp": { + "type": "date", + "format": "strict_date_hour_minute_second" + }, + "bucket_id": { + "type": "string", + "index": "not_analyzed" + }, + "file_id": { + "type": "string", + "index": "not_analyzed" + }, + "file_key": { + "type": "string", + "index": "not_analyzed" + }, + "unique_id": { + "type": "string", + "index": "not_analyzed" + }, + "visitor_id": { + "type": "string", + "index": "not_analyzed" + }, + "collection": { + "type": "string", + "index": "not_analyzed" + } + } + } + }, + "aliases": { + "events-stats-file-download": {} + } + } \ No newline at end of file diff --git a/elasticsearch/mappings/record-view/record-view-v1.json b/elasticsearch/mappings/record-view/record-view-v1.json new file mode 100644 index 0000000..bd6a035 --- /dev/null +++ b/elasticsearch/mappings/record-view/record-view-v1.json @@ -0,0 +1,53 @@ +{ + "template": "events-stats-record-view-*", + "settings": { + "index": { + "refresh_interval": "5s" + } + }, + "mappings": { + "record-view": { + "_source": { + "enabled": false + }, + "_all": { + "enabled": false + }, + "date_detection": false, + "numeric_detection": false, + "properties": { + "timestamp": { + "type": "date", + "format": "strict_date_hour_minute_second" + }, + "record_id": { + "type": "string", + "index": "not_analyzed" + }, + "pid_type": { + "type": "string", + "index": "not_analyzed" + }, + "pid_value": { + "type": "string", + "index": "not_analyzed" + }, + "labels": { + "type": "string", + "index": "not_analyzed" + }, + "visitor_id": { + "type": "string", + "index": "not_analyzed" + }, + "community": { + "type": "string", + "index": "not_analyzed" + } + } + } + }, + "aliases": { + "events-stats-record-view": {} + } + } diff --git a/nginx/Dockerfile b/nginx/Dockerfile index c2925a0..af65151 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -1,4 +1,4 @@ -FROM nginx:1.18 +FROM nginx:mainline-alpine EXPOSE 80 443 RUN rm /etc/nginx/conf.d/*.conf diff --git a/nginx/b2share.conf b/nginx/b2share.conf index 7d682a9..8f5ed14 100644 --- a/nginx/b2share.conf +++ b/nginx/b2share.conf @@ -2,6 +2,8 @@ server { listen 80; charset utf-8; + server_tokens off; + location /oai2d { proxy_pass http://b2share:5000/api/oai2d; proxy_set_header Host $host; @@ -23,8 +25,9 @@ server { charset utf-8; ssl_certificate /etc/ssl/b2share.crt; ssl_certificate_key /etc/ssl/b2share.key; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_ciphers HIGH:!aNULL:!MD5; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers !aNULL:!MD5:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; client_body_timeout 600s; diff --git a/upgrade.md b/upgrade.md new file mode 100644 index 0000000..343ee15 --- /dev/null +++ b/upgrade.md @@ -0,0 +1,82 @@ +# Upgrade guide for POSTGRESQL + +From 9.6 to 13 + +**Step 1: Get a database dump** + +With a running stack before upgrade: + +```bash +docker-compose exec postgres pg_dump --blobs --clean --dbname="${DBNAME}" --username="${DBUSER}" | gzip > dumpfile.sql.gz +``` + +**Step 2: DELETE old db** + +With a running stack before upgrade: + +```bash +docker container ls --filter name=_postgres_1 --format {{.ID}} +``` + +Variable `COMPOSE_PROJECT_NAME` is define in the .env file of the project. If the env variable is not define, the default value is the name of the folder where the .yml file is built. + +```bash +docker exec -i psql --user postgres -c 'DROP database ' +``` + +**Step 3: Upgrade postgres** + +New version of postgres is defined in the `docker-compose.yml` file. Thus, you only need to run: + +```bash +docker-compose up -d postgres +``` + +**Step 4: Recreate database** + +With a running stack after upgrade: + +Get ID of the recreated container (Same as in step 2): + +```bash +docker container ls --filter name=_postgres_1 --format {{.ID}} +``` + +```bash +docker exec -i psql --user postgres -c 'CREATE database ' +``` + +**Step 5: Restore data** + +With a running stack after upgrade: + +```bash +# Uncompressed .sql file: +docker exec -i psql --user < +# Gzip compressed .sql.gz file: +zcat | docker exec -i psql --user +``` + +Now the database should be upgraded to version 13 + +**If there are issues** it is a good idea to reindex elasticsearch indices: + +Get B2SHARE container ID: + +```bash +docker container ls --filter name=_b2share_1 --format {{.ID}} +``` + +With container ID in hand, run: + +```bash +docker exec -it bash +# These commands executed in B2SHARE container +b2share index destroy --force --yes-i-know +b2share index init --force +b2share index queue init purge + +b2share index reindex --yes-i-know +b2share index run + +```