Feature/es backup

k8s-aws/backups/Dockerfile

@@ -30,6 +30,9 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2
 RUN unzip awscliv2.zip
 RUN ./aws/install
 
+# Install Python dependencies
+RUN pip install boto3 click requests requests_aws4auth
+
 # Cron
 RUN mkdir /cronjobs && mkdir /cronjobs/backups
 
@@ -39,7 +42,7 @@ RUN mkdir /cronjobs/backups/mongo-ct
 RUN mkdir /cronjobs/backups/postgres
 RUN mkdir /cronjobs/backups/neo4j
 COPY automongobackup.sh /cronjobs/automongobackup.sh
-COPY autoelasticbackup.sh /cronjobs/autoelasticbackup.sh
+COPY autoelasticbackup.py /cronjobs/autoelasticbackup.py
 COPY autopostgresbackup.sh /cronjobs/autopostgresbackup.sh
 COPY autoneobackup.sh /cronjobs/autoneobackup.sh
 

k8s-aws/backups/autoelasticbackup.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+import os
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+import boto3
+import click
+import requests
+from requests.model import Response
+from requests_aws4auth import AWS4Auth
+
+
+@click.command()
+@click.option('--host', type=str, help='Elastic Search Host.')
+@click.option('--region', type=str, default="us-east-1", help='AWS region.')
+@click.option('--snapshot', type=str, help='Snapshot name.')
+@click.option('--role_arn', type=str, help='Snapshot IAM Role.')
+@click.option('--bucket', type=str, help='Backup S3 Bucket.')
+@click.option('--base_path', type=str, required=False, help='S3 Prefix.')
+def cli(host: str, region: str, snapshot: str, role_arn: str, bucket: str, base_path: Optional[str] = None) -> None:
+    """Main function to trigger Elastic Search Backup"""
+    service: str = 'es'
+    credentials = boto3.Session().get_credentials()
+    awsauth: AWS4Auth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service,
+                                 session_token=credentials.token)
+
+    settings: Dict[str, str] = {
+        "bucket": bucket,
+        "region": region,
+        "role_arn": role_arn
+    }
+    if base_path:
+        settings["base_path"] = base_path
+
+    register_snapshot(host, snapshot, awsauth, settings)
+    take_snapshot(host, snapshot, awsauth)
+
+
+def register_snapshot(host: str, snapshot: str, awsauth: str, settings: Dict[str, str]) -> None:
+    url: str = os.path.join(host, "_snapshot", snapshot)
+
+    payload: Dict[str, Any] = {
+        "type": "s3",
+        "settings": settings
+    }
+
+    headers = {"Content-Type": "application/json"}
+
+    r = requests.put(url, auth=awsauth, json=payload, headers=headers)
+
+    if r.status_code != 200:
+        click.echo("WARNING: Cannot register snapshot.")
+        click.echo(r.text)
+
+    else:
+        click.echo("Successfully registered snapshot.")
+
+
+def take_snapshot(host: str, snapshot: str, awsauth: str) -> None:
+    name: str = f"{datetime.utcnow()}".replace(" ", "_")
+    url: str = os.path.join(host, "_snapshot", snapshot, name)
+
+    r: Response = requests.put(url, auth=awsauth)
+
+    if r.status_code != 200:
+        raise RuntimeError(r.text)
+    else:
+        click.echo("Successfully started taking snapshot.")
+
+
+if __name__ == "__main__":
+    cli()

k8s-aws/backups/elasticsearch-backup.yaml

@@ -12,19 +12,24 @@ spec:
         spec:
           containers:
             - name: kubecron
-              image: vizzuality/kubecron:2.0.0
+              image: vizzuality/kubecron:2.1.5
               imagePullPolicy: Always
               env:
-                - name: AWS_BACKUPS_BUCKET_URI
-                  valueFrom:
-                    secretKeyRef:
-                      name: backups
-                      key: AWS_BACKUPS_BUCKET_URI
                 - name: AWS_BACKUPS_BUCKET_NAME
                   valueFrom:
                     secretKeyRef:
                       name: backups
                       key: AWS_BACKUPS_BUCKET_NAME
+                - name: ES_URI
+                    valueFrom:
+                      secretKeyRef:
+                        name: backups
+                        key: ES_URI
+                - name: ES_BACKUP_IAM_ROLE
+                    valueFrom:
+                      secretKeyRef:
+                        name: backups
+                        key: ES_BACKUP_IAM_ROLE
               args:
                 - elasticsearch
           restartPolicy: OnFailure

k8s-aws/backups/entrypoint.sh

@@ -20,7 +20,12 @@ case "$1" in
         ;;
     elasticsearch)
         echo "Starting auto elastic backup"
-        /cronjobs/autoelasticbackup.sh || true
+        /cronjobs/autoelasticbackup.py --host "$ES_URI" \
+                                       --snapshot wri-api-backups-es7 \
+                                       --role_arn "$ES_BACKUP_IAM_ROLE" \
+                                       --bucket "$AWS_BACKUPS_BUCKET_NAME" \
+                                       --base_path elasticsearch-7 \
+                                       || true
         ;;
     mongo)
         echo "Starting auto mongo backup"