From 325a4c885a12f32daf5d974cd28e1381d05624f3 Mon Sep 17 00:00:00 2001 From: Thomas Maschler Date: Wed, 4 Nov 2020 22:59:28 -0500 Subject: [PATCH 1/2] backup script --- k8s-aws/backups/autoelasticbackup.py | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 k8s-aws/backups/autoelasticbackup.py diff --git a/k8s-aws/backups/autoelasticbackup.py b/k8s-aws/backups/autoelasticbackup.py new file mode 100644 index 00000000..73de673a --- /dev/null +++ b/k8s-aws/backups/autoelasticbackup.py @@ -0,0 +1,71 @@ +import os +from datetime import datetime + +import boto3 +import click +import requests +from requests.model import Response +from requests_aws4auth import AWS4Auth +from typing import Any, Dict, Optional + + +@click.command() +@click.option('--host', type=str, help='Elastic Search Host.') +@click.option('--region', type=str, default="us-east-1", help='AWS region.') +@click.option('--snapshot', type=str, help='Snapshot name.') +@click.option('--role_arn', type=str, help='Snapshot IAM Role.') +@click.option('--bucket', type=str, help='Backup S3 Bucket.') +@click.option('--base_path', type=str, required=False, help='S3 Prefix.') +def cli(host: str, region: str, snapshot: str, role_arn: str, bucket: str, base_path: Optional[str] = None) -> None: + """Main function to trigger Elastic Search Backup""" + service: str = 'es' + credentials = boto3.Session().get_credentials() + awsauth: AWS4Auth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, + session_token=credentials.token) + + settings: Dict[str, str] = { + "bucket": bucket, + "region": region, + "role_arn": role_arn + } + if base_path: + settings["base_path"] = base_path + + register_snapshot(host, snapshot, awsauth, settings) + take_snapshot(host, snapshot, awsauth) + + +def register_snapshot(host: str, snapshot: str, awsauth: str, settings: Dict[str, str]) -> None: + url: str = os.path.join(host, "_snapshot", snapshot) + + payload: Dict[str, Any] = { + "type": "s3", + "settings": settings + } + + headers = {"Content-Type": "application/json"} + + r = requests.put(url, auth=awsauth, json=payload, headers=headers) + + if r.status_code != 200: + click.echo("WARNING: Cannot register snapshot.") + click.echo(r.text) + + else: + click.echo("Successfully registered snapshot.") + + +def take_snapshot(host: str, snapshot: str, awsauth: str) -> None: + name: str = f"{datetime.utcnow()}".replace(" ", "_") + url: str = os.path.join(host, "_snapshot", snapshot, name) + + r: Response = requests.put(url, auth=awsauth) + + if r.status_code != 200: + raise RuntimeError(r.text) + else: + click.echo("Successfully started taking snapshot.") + + +if __name__ == "__main__": + cli() From f3c6db9fb565e599e1eb7b02a53068efc52cfa90 Mon Sep 17 00:00:00 2001 From: Thomas Maschler Date: Wed, 4 Nov 2020 23:52:06 -0500 Subject: [PATCH 2/2] update ES backup script --- k8s-aws/backups/Dockerfile | 5 ++++- k8s-aws/backups/autoelasticbackup.py | 3 ++- k8s-aws/backups/autoelasticbackup.sh | 22 ---------------------- k8s-aws/backups/elasticsearch-backup.yaml | 17 +++++++++++------ k8s-aws/backups/entrypoint.sh | 7 ++++++- 5 files changed, 23 insertions(+), 31 deletions(-) delete mode 100755 k8s-aws/backups/autoelasticbackup.sh diff --git a/k8s-aws/backups/Dockerfile b/k8s-aws/backups/Dockerfile index fffc7f0e..681a885c 100644 --- a/k8s-aws/backups/Dockerfile +++ b/k8s-aws/backups/Dockerfile @@ -30,6 +30,9 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2 RUN unzip awscliv2.zip RUN ./aws/install +# Install Python dependencies +RUN pip install boto3 click requests requests_aws4auth + # Cron RUN mkdir /cronjobs && mkdir /cronjobs/backups @@ -39,7 +42,7 @@ RUN mkdir /cronjobs/backups/mongo-ct RUN mkdir /cronjobs/backups/postgres RUN mkdir /cronjobs/backups/neo4j COPY automongobackup.sh /cronjobs/automongobackup.sh -COPY autoelasticbackup.sh /cronjobs/autoelasticbackup.sh +COPY autoelasticbackup.py /cronjobs/autoelasticbackup.py COPY autopostgresbackup.sh /cronjobs/autopostgresbackup.sh COPY autoneobackup.sh /cronjobs/autoneobackup.sh diff --git a/k8s-aws/backups/autoelasticbackup.py b/k8s-aws/backups/autoelasticbackup.py index 73de673a..85a6f63b 100644 --- a/k8s-aws/backups/autoelasticbackup.py +++ b/k8s-aws/backups/autoelasticbackup.py @@ -1,12 +1,13 @@ +#!/usr/bin/env python import os from datetime import datetime +from typing import Any, Dict, Optional import boto3 import click import requests from requests.model import Response from requests_aws4auth import AWS4Auth -from typing import Any, Dict, Optional @click.command() diff --git a/k8s-aws/backups/autoelasticbackup.sh b/k8s-aws/backups/autoelasticbackup.sh deleted file mode 100755 index 30ade702..00000000 --- a/k8s-aws/backups/autoelasticbackup.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -DATE=$(date +%Y-%m-%d_%Hh%Mm) -URL_PATH='http://elasticsearch.core.svc.cluster.local:9200/_snapshot/wri-api-backups/' -PARAMS='?wait_for_completion=true' -URL=$URL_PATH$DATE$PARAMS - -echo 'Creating snapshot repository' -curl -X PUT \ - http://elasticsearch.core.svc.cluster.local:9200/_snapshot/wri-api-backups/ \ - --header 'Content-Type: application/json' \ - -d '{ - "type": "s3", - "settings": { - "bucket": "'$AWS_BACKUPS_BUCKET_NAME'", - "base_path": "elasticsearch" - } -}' -echo '' -echo 'Uploading Elastic Backup to:' -echo $URL -curl -XPUT $URL \ - --header 'Content-Type: application/json' diff --git a/k8s-aws/backups/elasticsearch-backup.yaml b/k8s-aws/backups/elasticsearch-backup.yaml index eb5ab612..4fc959af 100644 --- a/k8s-aws/backups/elasticsearch-backup.yaml +++ b/k8s-aws/backups/elasticsearch-backup.yaml @@ -12,19 +12,24 @@ spec: spec: containers: - name: kubecron - image: vizzuality/kubecron:2.0.0 + image: vizzuality/kubecron:2.1.5 imagePullPolicy: Always env: - - name: AWS_BACKUPS_BUCKET_URI - valueFrom: - secretKeyRef: - name: backups - key: AWS_BACKUPS_BUCKET_URI - name: AWS_BACKUPS_BUCKET_NAME valueFrom: secretKeyRef: name: backups key: AWS_BACKUPS_BUCKET_NAME + - name: ES_URI + valueFrom: + secretKeyRef: + name: backups + key: ES_URI + - name: ES_BACKUP_IAM_ROLE + valueFrom: + secretKeyRef: + name: backups + key: ES_BACKUP_IAM_ROLE args: - elasticsearch restartPolicy: OnFailure diff --git a/k8s-aws/backups/entrypoint.sh b/k8s-aws/backups/entrypoint.sh index ae39872e..69257000 100755 --- a/k8s-aws/backups/entrypoint.sh +++ b/k8s-aws/backups/entrypoint.sh @@ -20,7 +20,12 @@ case "$1" in ;; elasticsearch) echo "Starting auto elastic backup" - /cronjobs/autoelasticbackup.sh || true + /cronjobs/autoelasticbackup.py --host "$ES_URI" \ + --snapshot wri-api-backups-es7 \ + --role_arn "$ES_BACKUP_IAM_ROLE" \ + --bucket "$AWS_BACKUPS_BUCKET_NAME" \ + --base_path elasticsearch-7 \ + || true ;; mongo) echo "Starting auto mongo backup"