feat: Add uploading of backup archive to S3

backup_github/main.py

@@ -14,7 +14,7 @@
     success,
 )
 from backup_github.parse_args import parse_args
-from backup_github.utils import count_sizes
+from backup_github.utils import count_sizes, upload_to_s3
 
 logging.basicConfig(level=logging.INFO)
 
@@ -47,6 +47,15 @@ def main():
             backup.backup_pulls()
             logging.info("Finish backup of pulls")
         success.labels(parsed_args.organization).set(1)
+        if parsed_args.bucket:
+            upload_to_s3(
+                parsed_args.ak,
+                parsed_args.sk,
+                parsed_args.endpoint,
+                parsed_args.output_dir,
+                parsed_args.bucket,
+                parsed_args.organization,
+            )
     except Exception as e:
         logging.error(e)
         success.labels(parsed_args.organization).set(0)

backup_github/parse_args.py

@@ -69,5 +69,25 @@ def parse_args(args=None) -> argparse.Namespace:
         dest="metrics_path",
         help="path for .prom file with metrics",
     )
+    parser.add_argument(
+        "--ak",
+        dest="ak",
+        help="AK for obs",
+    )
+    parser.add_argument(
+        "--sk",
+        dest="sk",
+        help="SK for obs",
+    )
+    parser.add_argument(
+        "--endpoint",
+        dest="endpoint",
+        help="endpoint for obs",
+    )
+    parser.add_argument(
+        "--bucket",
+        dest="bucket",
+        help="bucket name",
+    )
     parsed = parser.parse_args(args)
     return parsed

backup_github/utils.py

@@ -1,10 +1,14 @@
 import json
 import logging
 import os
+import shutil
 import subprocess
 import sys
+from datetime import datetime
 from pathlib import Path
 
+import boto3 as boto3
+
 
 def save_json(path, content):
     mode = "w" if os.path.exists(path) else "w+"
@@ -49,3 +53,24 @@ def count_sizes(output_dir):
         )
     meta = sum(p.stat().st_size for p in Path(output_dir).rglob("*")) - git
     return {"git": git, "meta": meta}
+
+
+def upload_to_s3(ak, sk, endpoint, backup_dir, bucket, organization):
+    session = boto3.session.Session()
+    s3 = session.client(
+        service_name="s3",
+        aws_access_key_id=ak,
+        aws_secret_access_key=sk,
+        endpoint_url=endpoint,
+        verify=False,
+    )
+    shutil.make_archive(base_name="backup_archive", format="gztar", root_dir=backup_dir)
+    resp = s3.upload_file(
+        "./backup_archive.tar.gz",
+        bucket,
+        f'{organization}-{datetime.now().strftime("%m-%d-%Y_%H-%M")}.tar.gz',
+    )
+    if resp.status >= 300:
+        logging.error(f"Uploading of backup failed, error message: {resp.errorMessage}")
+        raise Exception(resp.errorMessage)
+    logging.info("Backup is loaded")

pyproject.toml

@@ -10,6 +10,7 @@ packages = [{include = "backup_github"}]
 python = "^3.8.1"
 requests = "^2.28.2"
 prometheus_client = "^0.16.0"
+boto3 = "^1.28.64"
 
 
 [tool.poetry.dev-dependencies]