fix: upload to s3

backup_github/main.py

@@ -1,4 +1,3 @@
-import datetime
 import logging
 import sys
 from time import time
@@ -14,9 +13,8 @@
     registry,
     success,
 )
-from backup_github.obs import BackupObs
 from backup_github.parse_args import parse_args
-from backup_github.utils import count_sizes
+from backup_github.utils import count_sizes, upload_to_s3
 
 logging.basicConfig(level=logging.INFO)
 
@@ -49,15 +47,8 @@ def main():
             backup.backup_pulls()
             logging.info("Finish backup of pulls")
         success.labels(parsed_args.organization).set(1)
-        obs = BackupObs(
-            access_key_id=parsed_args.ak,
-            secret_access_key=parsed_args.sk,
-            server=parsed_args.endpoint,
-            path_style=True,
-            signature="v2",
-            is_signature_negotiation=True
-        )
-        obs.upload_backup(f'{parsed_args.organization}-{datetime.date}', parsed_args.output_dir, parsed_args.bucket)
+        if parsed_args.bucket:
+            upload_to_s3(parsed_args.ak, parsed_args.sk, parsed_args.output_dir, parsed_args.bucket, parsed_args.organization)
     except Exception as e:
         logging.error(e)
         success.labels(parsed_args.organization).set(0)

backup_github/utils.py

@@ -1,10 +1,14 @@
 import json
 import logging
 import os
+import shutil
 import subprocess
 import sys
+from datetime import datetime
 from pathlib import Path
 
+from obs import ObsClient
+
 
 def save_json(path, content):
     mode = "w" if os.path.exists(path) else "w+"
@@ -49,3 +53,26 @@ def count_sizes(output_dir):
         )
     meta = sum(p.stat().st_size for p in Path(output_dir).rglob("*")) - git
     return {"git": git, "meta": meta}
+
+
+def upload_to_s3(ak, sk, endpoint, backup_dir, bucket, organization):
+    obs = ObsClient(
+        access_key_id=ak,
+        secret_access_key=sk,
+        server=endpoint,
+        path_style=True,
+        signature="v2",
+        is_signature_negotiation=True
+    )
+    shutil.make_archive(
+        base_name='backup_archive',
+        format='gztar',
+        root_dir=backup_dir
+    )
+    resp = obs.putFile(bucket,
+                       f'{organization}-{datetime.now().strftime("%m-%d-%Y_%H-%M")}.tar.gz',
+                       './backup_archive.tar.gz')
+    if resp.status >= 300:
+        logging.error(f'Uploading of backup failed, error message: {resp.errorMessage}')
+        raise Exception(resp.errorMessage)
+    logging.info(f'Backup is loaded')

pyproject.toml

@@ -10,6 +10,7 @@ packages = [{include = "backup_github"}]
 python = "^3.8.1"
 requests = "^2.28.2"
 prometheus_client = "^0.16.0"
+huaweicloud-sdk-python-obs = "^3.21.8"
 
 
 [tool.poetry.dev-dependencies]