diff --git a/backup_github/main.py b/backup_github/main.py index 2d41935..689dbd4 100644 --- a/backup_github/main.py +++ b/backup_github/main.py @@ -14,7 +14,7 @@ success, ) from backup_github.parse_args import parse_args -from backup_github.utils import count_sizes +from backup_github.utils import count_sizes, upload_to_s3 logging.basicConfig(level=logging.INFO) @@ -47,6 +47,15 @@ def main(): backup.backup_pulls() logging.info("Finish backup of pulls") success.labels(parsed_args.organization).set(1) + if parsed_args.bucket: + upload_to_s3( + parsed_args.ak, + parsed_args.sk, + parsed_args.endpoint, + parsed_args.output_dir, + parsed_args.bucket, + parsed_args.organization, + ) except Exception as e: logging.error(e) success.labels(parsed_args.organization).set(0) diff --git a/backup_github/parse_args.py b/backup_github/parse_args.py index e49fbbe..3270475 100644 --- a/backup_github/parse_args.py +++ b/backup_github/parse_args.py @@ -69,5 +69,25 @@ def parse_args(args=None) -> argparse.Namespace: dest="metrics_path", help="path for .prom file with metrics", ) + parser.add_argument( + "--ak", + dest="ak", + help="AK for obs", + ) + parser.add_argument( + "--sk", + dest="sk", + help="SK for obs", + ) + parser.add_argument( + "--endpoint", + dest="endpoint", + help="endpoint for obs", + ) + parser.add_argument( + "--bucket", + dest="bucket", + help="bucket name", + ) parsed = parser.parse_args(args) return parsed diff --git a/backup_github/utils.py b/backup_github/utils.py index 5348581..4bc2449 100644 --- a/backup_github/utils.py +++ b/backup_github/utils.py @@ -1,10 +1,14 @@ import json import logging import os +import shutil import subprocess import sys +from datetime import datetime from pathlib import Path +import boto3 as boto3 + def save_json(path, content): mode = "w" if os.path.exists(path) else "w+" @@ -49,3 +53,24 @@ def count_sizes(output_dir): ) meta = sum(p.stat().st_size for p in Path(output_dir).rglob("*")) - git return {"git": git, "meta": meta} + + +def upload_to_s3(ak, sk, endpoint, backup_dir, bucket, organization): + session = boto3.session.Session() + s3 = session.client( + service_name="s3", + aws_access_key_id=ak, + aws_secret_access_key=sk, + endpoint_url=endpoint, + verify=False, + ) + shutil.make_archive(base_name="backup_archive", format="gztar", root_dir=backup_dir) + resp = s3.upload_file( + "./backup_archive.tar.gz", + bucket, + f'{organization}-{datetime.now().strftime("%m-%d-%Y_%H-%M")}.tar.gz', + ) + if resp.status >= 300: + logging.error(f"Uploading of backup failed, error message: {resp.errorMessage}") + raise Exception(resp.errorMessage) + logging.info("Backup is loaded") diff --git a/pyproject.toml b/pyproject.toml index d8dd554..e83ae71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ packages = [{include = "backup_github"}] python = "^3.8.1" requests = "^2.28.2" prometheus_client = "^0.16.0" +boto3 = "^1.28.64" [tool.poetry.dev-dependencies] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 94caedc..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pytest -requests -requests-mock \ No newline at end of file