From 27156b598a722e953a9c1287703e320187d53e19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20L=C3=B6sche?= Date: Sun, 14 Jul 2024 14:04:12 +0200 Subject: [PATCH] Add postgresql string --- Dockerfile | 1 + README.md | 34 +++++++--- fixbackup/__init__.py | 2 +- fixbackup/__main__.py | 16 ++++- fixbackup/args.py | 16 ++++- fixbackup/backup/__init__.py | 20 +++++- fixbackup/backup/postgresql.py | 110 +++++++++++++++++++++++++++++++++ fixbackup/logger.py | 15 ++--- pyproject.toml | 2 +- requirements-test.txt | 70 ++++++++++----------- requirements.txt | 10 +-- tox.ini | 2 +- 12 files changed, 233 insertions(+), 65 deletions(-) create mode 100644 fixbackup/backup/postgresql.py diff --git a/Dockerfile b/Dockerfile index ff99036..571d699 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,6 +68,7 @@ RUN groupadd -g "${PGID:-0}" -o fix \ python3-pip \ redis-tools \ mysql-client \ + postgresql-client \ && dpkg -i /usr/local/tmp/arangodb3-client_*.deb \ && ln -s /usr/bin/busybox /usr/local/bin/vi \ && ln -s /usr/bin/busybox /usr/local/bin/wget \ diff --git a/README.md b/README.md index be4c006..c749110 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,25 @@ -# `fixbackup` - FIX Database Backup System +# `fixbackup` - Fix Database Backup System + +A wrapper tool packaged as a container, that runs as a job, creates backups of various databases, and uploads them to an S3 bucket. ## Usage ```bash -usage: fixbackup [-h] [--backup-directory BACKUP_DIRECTORY] [--verbose | --trace | --quiet] [--s3-bucket S3_BUCKET] --type {daily,weekly,monthly,yearly} - [--set-lifecycle-policy] [--redis-host REDIS_HOST] [--redis-port REDIS_PORT] [--redis-username REDIS_USERNAME] - [--redis-password REDIS_PASSWORD] [--redis-database-number REDIS_DATABASE_NUMBER] [--redis-cli-args REDIS_CLI_ARGS] - [--mysql-host MYSQL_HOST] [--mysql-port MYSQL_PORT] [--mysql-user MYSQL_USER] [--mysql-password MYSQL_PASSWORD] - [--mysqldump-args MYSQLDUMP_ARGS] [--arangodb-host ARANGODB_HOST] [--arangodb-port ARANGODB_PORT] [--arangodb-username ARANGODB_USERNAME] - [--arangodb-password ARANGODB_PASSWORD] [--arangodb-database ARANGODB_DATABASE] [--arangodump-args ARANGODUMP_ARGS] +usage: fixbackup [-h] [--backup-directory BACKUP_DIRECTORY] [-n ENVIRONMENT] [--sleep] [--verbose | --trace | --quiet] [--s3-bucket S3_BUCKET] --type {daily,weekly,monthly,yearly} [--set-lifecycle-policy] [--redis-host REDIS_HOST] + [--redis-port REDIS_PORT] [--redis-username REDIS_USERNAME] [--redis-password REDIS_PASSWORD] [--redis-database-number REDIS_DATABASE_NUMBER] [--redis-cli-args REDIS_CLI_ARGS] [--redis-tls] [--redis-tls-insecure] + [--mysql-host MYSQL_HOST] [--mysql-port MYSQL_PORT] [--mysql-user MYSQL_USER] [--mysql-password MYSQL_PASSWORD] [--mysql-database MYSQL_DATABASE] [--mysqldump-args MYSQLDUMP_ARGS] [--pg-host PG_HOST] + [--pg-port PG_PORT] [--pg-user PG_USER] [--pg-password PG_PASSWORD] [--pg-database PG_DATABASE] [--pg-dump-args PG_DUMP_ARGS] [--arangodb-host ARANGODB_HOST] [--arangodb-port ARANGODB_PORT] + [--arangodb-username ARANGODB_USERNAME] [--arangodb-password ARANGODB_PASSWORD] [--arangodb-database ARANGODB_DATABASE] [--arangodump-args ARANGODUMP_ARGS] [--arangodb-tls] -FIX Database Backup System +Fix Database Backup System options: -h, --help show this help message and exit --backup-directory BACKUP_DIRECTORY Directory where backups are created + -n ENVIRONMENT, --name ENVIRONMENT + Name of the environment + --sleep Don't do anything, just sleep forever --verbose, -v Verbose logging --trace Trage logging --quiet Only log errors @@ -37,6 +41,8 @@ options: Redis database number --redis-cli-args REDIS_CLI_ARGS Extra arguments to pass to redis-cli + --redis-tls Redis uses TLS + --redis-tls-insecure Redis uses TLS without verifying the certificate --mysql-host MYSQL_HOST MySQL host --mysql-port MYSQL_PORT @@ -45,8 +51,19 @@ options: MySQL user --mysql-password MYSQL_PASSWORD MySQL password + --mysql-database MYSQL_DATABASE + MySQL database --mysqldump-args MYSQLDUMP_ARGS Extra arguments to pass to mysqldump + --pg-host PG_HOST PostgreSQL host + --pg-port PG_PORT PostgreSQL port + --pg-user PG_USER PostgreSQL user + --pg-password PG_PASSWORD + PostgreSQL password + --pg-database PG_DATABASE + PostgreSQL database + --pg-dump-args PG_DUMP_ARGS + Extra arguments to pass to pg_dump --arangodb-host ARANGODB_HOST ArangoDB host --arangodb-port ARANGODB_PORT @@ -59,4 +76,5 @@ options: ArangoDB database to dump --arangodump-args ARANGODUMP_ARGS Extra arguments to pass to arangodump + --arangodb-tls ArangoDB uses TLS ``` diff --git a/fixbackup/__init__.py b/fixbackup/__init__.py index 546e629..f02fa87 100644 --- a/fixbackup/__init__.py +++ b/fixbackup/__init__.py @@ -11,4 +11,4 @@ __author__ = "Some Engineering Inc." __license__ = "Apache 2.0" __copyright__ = "Copyright © 2023 Some Engineering Inc." -__version__ = "0.0.7" +__version__ = "0.0.10" diff --git a/fixbackup/__main__.py b/fixbackup/__main__.py index 8b8e472..bf16414 100644 --- a/fixbackup/__main__.py +++ b/fixbackup/__main__.py @@ -1,5 +1,6 @@ import sys import os +import time from pathlib import Path from typing import List from .logger import add_args as logging_add_args, log @@ -12,11 +13,24 @@ def main() -> None: args = parse_args([logging_add_args, s3_add_args, *backup_add_args]) exit_code = 0 - log.info("Starting FIX Databases Backup System") + log.info("Starting Fix Databases Backup System") if not verify_binaries(): sys.exit(1) + if args.sleep: + # This option is used to keep the container running for debugging purposes. + # It allows you to connect to it inside of e.g. a K8s environment + # and manually test the backup process. Alternatively, you could + # override the entrypoint of the container and sleep indefinitely. + log.info("Sleeping forever") + try: + while True: + time.sleep(300) + finally: + log.info("Shutdown complete") + sys.exit(0) + backup_directory = Path(args.backup_directory) rmdir_backup_directory = True if backup_directory.exists() and not backup_directory.is_dir(): diff --git a/fixbackup/args.py b/fixbackup/args.py index 87028ef..e44da6c 100644 --- a/fixbackup/args.py +++ b/fixbackup/args.py @@ -4,7 +4,7 @@ def parse_args(add_args: List[Callable[[ArgumentParser], None]]) -> Namespace: - arg_parser = ArgumentParser(prog="fixbackup", description="FIX Database Backup System") + arg_parser = ArgumentParser(prog="fixbackup", description="Fix Database Backup and Restore System") arg_parser.add_argument( "--backup-directory", help="Directory where backups are created", @@ -19,6 +19,20 @@ def parse_args(add_args: List[Callable[[ArgumentParser], None]]) -> Namespace: help="Name of the environment", default=os.getenv("FIX_ENVIRONMENT", "dev"), ) + arg_parser.add_argument( + "--sleep", + help="Don't do anything, just sleep forever", + dest="sleep", + action="store_true", + default=False, + ) + arg_parser.add_argument( + "--restore", + help="Restore databases from directory", + dest="restore", + action="store_true", + default=False, + ) for add_arg in add_args: add_arg(arg_parser) diff --git a/fixbackup/backup/__init__.py b/fixbackup/backup/__init__.py index ae58885..6bfeb26 100644 --- a/fixbackup/backup/__init__.py +++ b/fixbackup/backup/__init__.py @@ -4,10 +4,11 @@ from typing import List, Tuple from .redis import backup as redis_backup, add_args as redis_add_args from .mysql import backup as mysql_backup, add_args as mysql_add_args +from .postgresql import backup as pg_backup, add_args as postgresql_add_args from .arangodb import backup as arangodb_backup, add_args as arangodb_add_args from ..utils import valid_hostname, valid_ip, valid_dbname -add_args = [redis_add_args, mysql_add_args, arangodb_add_args] +add_args = [redis_add_args, mysql_add_args, postgresql_add_args, arangodb_add_args] def backup(args: Namespace, backup_directory: Path) -> Tuple[List[Path], bool]: @@ -39,6 +40,19 @@ def backup(args: Namespace, backup_directory: Path) -> Tuple[List[Path], bool]: else: all_success = False + if args.pg_host and (valid_hostname(args.pg_host) or valid_ip(args.pg_host)): + if args.pg_database: + db = str(args.pg_database) + if not valid_dbname(db): + raise ValueError(f"Invalid database name: {db}") + else: + db = "all" + pg_backup_file = backup_directory / f"{environment}-{date_prefix}-postgresql-{args.pg_host}-{db}.sql.gz" + if pg_backup(args, pg_backup_file): + result.append(pg_backup_file) + else: + all_success = False + if args.arangodb_host and (valid_hostname(args.arangodb_host)): if args.arangodb_database: db = str(args.arangodb_database) @@ -46,7 +60,9 @@ def backup(args: Namespace, backup_directory: Path) -> Tuple[List[Path], bool]: raise ValueError(f"Invalid database name: {db}") else: db = "all" - arangodb_backup_file = backup_directory / f"{environment}-{date_prefix}-arangodb-{args.arangodb_host}-{db}.tar.gz" + arangodb_backup_file = ( + backup_directory / f"{environment}-{date_prefix}-arangodb-{args.arangodb_host}-{db}.tar.gz" + ) if arangodb_backup(args, arangodb_backup_file): result.append(arangodb_backup_file) else: diff --git a/fixbackup/backup/postgresql.py b/fixbackup/backup/postgresql.py new file mode 100644 index 0000000..323ff51 --- /dev/null +++ b/fixbackup/backup/postgresql.py @@ -0,0 +1,110 @@ +import os +import subprocess +from pathlib import Path +from argparse import ArgumentParser, Namespace +from ..utils import BackupFile +from ..logger import log + + +def add_args(arg_parser: ArgumentParser) -> None: + arg_parser.add_argument( + "--pg-host", + help="PostgreSQL host", + dest="pg_host", + type=str, + default=os.getenv("PG_HOST"), + ) + + arg_parser.add_argument( + "--pg-port", + help="PostgreSQL port", + dest="pg_port", + type=int, + default=os.getenv("PG_PORT", 5432), + ) + + arg_parser.add_argument( + "--pg-user", + help="PostgreSQL user", + dest="pg_user", + type=str, + default=os.getenv("PG_USER", "postgres"), + ) + + arg_parser.add_argument( + "--pg-password", + help="PostgreSQL password", + dest="pg_password", + type=str, + default=os.getenv("PG_PASSWORD"), + ) + + arg_parser.add_argument( + "--pg-database", + help="PostgreSQL database", + dest="pg_database", + type=str, + default=os.getenv("PG_DATABASE"), + ) + + arg_parser.add_argument( + "--pg-dump-args", + help="Extra arguments to pass to pg_dump", + dest="pg_dump_args", + action="append", + default=[], + ) + + +def backup(args: Namespace, backup_file_path: Path, timeout: int = 900, compress: bool = True) -> bool: + log.info("Starting PostgreSQL backup...") + + if not args.pg_host: + return False + + env = os.environ.copy() + command = [ + "pg_dump", + "-w", + "-c", + "--if-exists", + "--inserts", + "-h", + str(args.pg_host), + "-p", + str(args.pg_port), + "-U", + str(args.pg_user), + *args.pg_dump_args, + ] + if args.pg_database: + command.append("-d") + command.append(args.pg_database) + else: + command[0] = "pg_dumpall" + + if args.pg_password: + env["PGPASSWORD"] = args.pg_password + + log.debug(f"Running command: {' '.join(command)}") + + try: + with BackupFile(backup_file_path, compress) as backup_fd: + process = subprocess.Popen(command, stdout=backup_fd, stderr=subprocess.PIPE, env=env) + _, stderr = process.communicate(timeout=timeout) + + if process.returncode == 0: + log.info(f"PostgreSQL backup completed successfully. Saved to {backup_file_path}") + if stderr: + log.debug(stderr.decode().strip()) + return True + else: + log.error(f"PostgreSQL backup failed with return code: {process.returncode}") + if stderr: + log.error(stderr.decode().strip()) + except subprocess.TimeoutExpired: + log.error(f"PostgreSQL backup failed with timeout after {timeout} seconds") + process.kill() + process.communicate() + + return False diff --git a/fixbackup/logger.py b/fixbackup/logger.py index ee093ad..3703a5a 100644 --- a/fixbackup/logger.py +++ b/fixbackup/logger.py @@ -126,20 +126,15 @@ def log_to_root(message: str, *args: Any, **kwargs: Any) -> None: class FixLogger(Logger): - def debug2(self, msg: str, *args: Any, **kwargs: Any) -> None: - ... + def debug2(self, msg: str, *args: Any, **kwargs: Any) -> None: ... - def debug3(self, msg: str, *args: Any, **kwargs: Any) -> None: - ... + def debug3(self, msg: str, *args: Any, **kwargs: Any) -> None: ... - def debug4(self, msg: str, *args: Any, **kwargs: Any) -> None: - ... + def debug4(self, msg: str, *args: Any, **kwargs: Any) -> None: ... - def debug5(self, msg: str, *args: Any, **kwargs: Any) -> None: - ... + def debug5(self, msg: str, *args: Any, **kwargs: Any) -> None: ... - def trace(self, msg: str, *args: Any, **kwargs: Any) -> None: - ... + def trace(self, msg: str, *args: Any, **kwargs: Any) -> None: ... def get_fix_logger(name: Optional[str] = None) -> FixLogger: diff --git a/pyproject.toml b/pyproject.toml index 6ae3274..9543c87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "fixbackup" -version = "0.0.7" +version = "0.0.10" authors = [{name="Some Engineering Inc."}] description = "FIX Database Backup System" license = {file="LICENSE"} diff --git a/requirements-test.txt b/requirements-test.txt index 2b8e084..17e79cb 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,16 +1,16 @@ -astroid==3.0.0 +astroid==3.2.3 # via pylint -attrs==23.1.0 +attrs==23.2.0 # via hypothesis -black==23.9.1 +black==24.4.2 # via fixbackup (pyproject.toml) -boto3==1.28.63 +boto3==1.34.144 # via fixbackup (pyproject.toml) -botocore==1.31.63 +botocore==1.34.144 # via # boto3 # s3transfer -cachetools==5.3.1 +cachetools==5.4.0 # via tox chardet==5.2.0 # via tox @@ -18,27 +18,27 @@ click==8.1.7 # via black colorama==0.4.6 # via tox -coverage[toml]==7.3.2 +coverage[toml]==7.6.0 # via # fixbackup (pyproject.toml) # pytest-cov -dill==0.3.7 +dill==0.3.8 # via pylint -distlib==0.3.7 +distlib==0.3.8 # via virtualenv -filelock==3.12.4 +filelock==3.15.4 # via # tox # virtualenv -flake8==6.1.0 +flake8==7.1.0 # via # fixbackup (pyproject.toml) # pep8-naming -hypothesis==6.87.4 +hypothesis==6.108.2 # via fixbackup (pyproject.toml) iniconfig==2.0.0 # via pytest -isort==5.12.0 +isort==5.13.2 # via pylint jmespath==1.0.1 # via @@ -48,68 +48,68 @@ mccabe==0.7.0 # via # flake8 # pylint -mypy==1.6.0 +mypy==1.10.1 # via fixbackup (pyproject.toml) mypy-extensions==1.0.0 # via # black # mypy -packaging==23.2 +packaging==24.1 # via # black # pyproject-api # pytest # tox -pathspec==0.11.2 +pathspec==0.12.1 # via black -pep8-naming==0.13.3 +pep8-naming==0.14.1 # via fixbackup (pyproject.toml) -platformdirs==3.11.0 +platformdirs==4.2.2 # via # black # pylint # tox # virtualenv -pluggy==1.3.0 +pluggy==1.5.0 # via # pytest # tox -pycodestyle==2.11.0 +pycodestyle==2.12.0 # via flake8 -pyflakes==3.1.0 +pyflakes==3.2.0 # via flake8 -pylint==3.0.1 +pylint==3.2.5 # via fixbackup (pyproject.toml) -pyproject-api==1.6.1 +pyproject-api==1.7.1 # via tox -pytest==7.4.2 +pytest==8.2.2 # via # fixbackup (pyproject.toml) # pytest-asyncio # pytest-cov -pytest-asyncio==0.21.1 +pytest-asyncio==0.23.7 # via fixbackup (pyproject.toml) -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via fixbackup (pyproject.toml) -pytest-runner==6.0.0 +pytest-runner==6.0.1 # via fixbackup (pyproject.toml) -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via botocore -s3transfer==0.7.0 +s3transfer==0.10.2 # via boto3 six==1.16.0 # via python-dateutil sortedcontainers==2.4.0 # via hypothesis -tomlkit==0.12.1 +tomlkit==0.13.0 # via pylint -tox==4.11.3 +tox==4.16.0 # via fixbackup (pyproject.toml) -typing-extensions==4.8.0 +typing-extensions==4.12.2 # via mypy -urllib3==2.0.7 +urllib3==2.2.2 # via botocore -virtualenv==20.24.5 +virtualenv==20.26.3 # via tox -wheel==0.41.2 +wheel==0.43.0 # via fixbackup (pyproject.toml) diff --git a/requirements.txt b/requirements.txt index 699b539..88d2235 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -boto3==1.28.63 +boto3==1.34.144 # via fixbackup (pyproject.toml) -botocore==1.31.63 +botocore==1.34.144 # via # boto3 # s3transfer @@ -8,11 +8,11 @@ jmespath==1.0.1 # via # boto3 # botocore -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via botocore -s3transfer==0.7.0 +s3transfer==0.10.2 # via boto3 six==1.16.0 # via python-dateutil -urllib3==2.0.7 +urllib3==2.2.2 # via botocore diff --git a/tox.ini b/tox.ini index 4e86489..ea4c504 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ env_list = syntax, tests, black, flake8, mypy [flake8] max-line-length = 120 exclude = .git,.tox,__pycache__,.idea,.pytest_cache,venv -ignore = F401, F403, F405, F811, E722, N806, N813, E266, W503, E203 +ignore = F401, F403, F405, E704, F811, E722, N806, N813, E266, W503, E203 [pytest] testpaths = test