Add postgresql string

Author: lloesche

Dockerfile

@@ -68,6 +68,7 @@ RUN groupadd -g "${PGID:-0}" -o fix \
         python3-pip \
         redis-tools \
         mysql-client \
+        postgresql-client \
     && dpkg -i /usr/local/tmp/arangodb3-client_*.deb \
     && ln -s /usr/bin/busybox /usr/local/bin/vi \
     && ln -s /usr/bin/busybox /usr/local/bin/wget \

README.md

@@ -1,21 +1,32 @@
-# `fixbackup` - FIX Database Backup System
+# `fixbackup` - Fix Database Backup and Restore System
+
+A wrapper tool packaged as a container, that runs as a job, creates and restores backups of various databases, and uploads them to an S3 bucket.
+
+## Docker
+
+```
+docker run --it --rm -v /path/to/backups:/backups someengineering/fixbackup --type daily ...
+```
 
 ## Usage
 
 ```bash
-usage: fixbackup [-h] [--backup-directory BACKUP_DIRECTORY] [--verbose | --trace | --quiet] [--s3-bucket S3_BUCKET] --type {daily,weekly,monthly,yearly}
-                 [--set-lifecycle-policy] [--redis-host REDIS_HOST] [--redis-port REDIS_PORT] [--redis-username REDIS_USERNAME]
-                 [--redis-password REDIS_PASSWORD] [--redis-database-number REDIS_DATABASE_NUMBER] [--redis-cli-args REDIS_CLI_ARGS]
-                 [--mysql-host MYSQL_HOST] [--mysql-port MYSQL_PORT] [--mysql-user MYSQL_USER] [--mysql-password MYSQL_PASSWORD]
-                 [--mysqldump-args MYSQLDUMP_ARGS] [--arangodb-host ARANGODB_HOST] [--arangodb-port ARANGODB_PORT] [--arangodb-username ARANGODB_USERNAME]
-                 [--arangodb-password ARANGODB_PASSWORD] [--arangodb-database ARANGODB_DATABASE] [--arangodump-args ARANGODUMP_ARGS]
+usage: fixbackup [-h] [--backup-directory BACKUP_DIRECTORY] [-n ENVIRONMENT] [--sleep] [--restore] [--verbose | --trace | --quiet] [--s3-bucket S3_BUCKET] --type {daily,weekly,monthly,yearly} [--set-lifecycle-policy] [--redis-host REDIS_HOST]
+                 [--redis-port REDIS_PORT] [--redis-username REDIS_USERNAME] [--redis-password REDIS_PASSWORD] [--redis-database-number REDIS_DATABASE_NUMBER] [--redis-cli-args REDIS_CLI_ARGS] [--redis-tls] [--redis-tls-insecure] [--mysql-host MYSQL_HOST]
+                 [--mysql-port MYSQL_PORT] [--mysql-user MYSQL_USER] [--mysql-password MYSQL_PASSWORD] [--mysql-database MYSQL_DATABASE] [--mysqldump-args MYSQLDUMP_ARGS] [--pg-host PG_HOST] [--pg-port PG_PORT] [--pg-user PG_USER] [--pg-password PG_PASSWORD]
+                 [--pg-database PG_DATABASE] [--pg-dump-args PG_DUMP_ARGS] [--arangodb-host ARANGODB_HOST] [--arangodb-port ARANGODB_PORT] [--arangodb-username ARANGODB_USERNAME] [--arangodb-password ARANGODB_PASSWORD] [--arangodb-database ARANGODB_DATABASE]
+                 [--arangodump-args ARANGODUMP_ARGS] [--arangodb-tls]
 
-FIX Database Backup System
+Fix Database Backup and Restore System
 
 options:
   -h, --help            show this help message and exit
   --backup-directory BACKUP_DIRECTORY
                         Directory where backups are created
+  -n ENVIRONMENT, --name ENVIRONMENT
+                        Name of the environment
+  --sleep               Don't do anything, just sleep forever
+  --restore             Restore databases from directory
   --verbose, -v         Verbose logging
   --trace               Trage logging
   --quiet               Only log errors
@@ -37,6 +48,8 @@ options:
                         Redis database number
   --redis-cli-args REDIS_CLI_ARGS
                         Extra arguments to pass to redis-cli
+  --redis-tls           Redis uses TLS
+  --redis-tls-insecure  Redis uses TLS without verifying the certificate
   --mysql-host MYSQL_HOST
                         MySQL host
   --mysql-port MYSQL_PORT
@@ -45,8 +58,19 @@ options:
                         MySQL user
   --mysql-password MYSQL_PASSWORD
                         MySQL password
+  --mysql-database MYSQL_DATABASE
+                        MySQL database
   --mysqldump-args MYSQLDUMP_ARGS
                         Extra arguments to pass to mysqldump
+  --pg-host PG_HOST     PostgreSQL host
+  --pg-port PG_PORT     PostgreSQL port
+  --pg-user PG_USER     PostgreSQL user
+  --pg-password PG_PASSWORD
+                        PostgreSQL password
+  --pg-database PG_DATABASE
+                        PostgreSQL database
+  --pg-dump-args PG_DUMP_ARGS
+                        Extra arguments to pass to pg_dump
   --arangodb-host ARANGODB_HOST
                         ArangoDB host
   --arangodb-port ARANGODB_PORT
@@ -59,4 +83,5 @@ options:
                         ArangoDB database to dump
   --arangodump-args ARANGODUMP_ARGS
                         Extra arguments to pass to arangodump
+  --arangodb-tls        ArangoDB uses TLS
 ```

fixbackup/__init__.py

@@ -11,4 +11,4 @@
 __author__ = "Some Engineering Inc."
 __license__ = "Apache 2.0"
 __copyright__ = "Copyright © 2023 Some Engineering Inc."
-__version__ = "0.0.7"
+__version__ = "0.0.10"

fixbackup/__main__.py

@@ -1,5 +1,6 @@
 import sys
 import os
+import time
 from pathlib import Path
 from typing import List
 from .logger import add_args as logging_add_args, log
@@ -12,11 +13,24 @@
 def main() -> None:
     args = parse_args([logging_add_args, s3_add_args, *backup_add_args])
     exit_code = 0
-    log.info("Starting FIX Databases Backup System")
+    log.info("Starting Fix Databases Backup System")
 
     if not verify_binaries():
         sys.exit(1)
 
+    if args.sleep:
+        # This option is used to keep the container running for debugging purposes.
+        # It allows you to connect to it inside of e.g. a K8s environment
+        # and manually test the backup process. Alternatively, you could
+        # override the entrypoint of the container and sleep indefinitely.
+        log.info("Sleeping forever")
+        try:
+            while True:
+                time.sleep(300)
+        finally:
+            log.info("Shutdown complete")
+            sys.exit(0)
+
     backup_directory = Path(args.backup_directory)
     rmdir_backup_directory = True
     if backup_directory.exists() and not backup_directory.is_dir():

fixbackup/args.py

@@ -4,7 +4,7 @@
 
 
 def parse_args(add_args: List[Callable[[ArgumentParser], None]]) -> Namespace:
-    arg_parser = ArgumentParser(prog="fixbackup", description="FIX Database Backup System")
+    arg_parser = ArgumentParser(prog="fixbackup", description="Fix Database Backup and Restore System")
     arg_parser.add_argument(
         "--backup-directory",
         help="Directory where backups are created",
@@ -19,6 +19,20 @@ def parse_args(add_args: List[Callable[[ArgumentParser], None]]) -> Namespace:
         help="Name of the environment",
         default=os.getenv("FIX_ENVIRONMENT", "dev"),
     )
+    arg_parser.add_argument(
+        "--sleep",
+        help="Don't do anything, just sleep forever",
+        dest="sleep",
+        action="store_true",
+        default=False,
+    )
+    arg_parser.add_argument(
+        "--restore",
+        help="Restore databases from directory",
+        dest="restore",
+        action="store_true",
+        default=False,
+    )
 
     for add_arg in add_args:
         add_arg(arg_parser)

fixbackup/backup/__init__.py

@@ -4,10 +4,11 @@
 from typing import List, Tuple
 from .redis import backup as redis_backup, add_args as redis_add_args
 from .mysql import backup as mysql_backup, add_args as mysql_add_args
+from .postgresql import backup as pg_backup, add_args as postgresql_add_args
 from .arangodb import backup as arangodb_backup, add_args as arangodb_add_args
 from ..utils import valid_hostname, valid_ip, valid_dbname
 
-add_args = [redis_add_args, mysql_add_args, arangodb_add_args]
+add_args = [redis_add_args, mysql_add_args, postgresql_add_args, arangodb_add_args]
 
 
 def backup(args: Namespace, backup_directory: Path) -> Tuple[List[Path], bool]:
@@ -39,14 +40,29 @@ def backup(args: Namespace, backup_directory: Path) -> Tuple[List[Path], bool]:
         else:
             all_success = False
 
+    if args.pg_host and (valid_hostname(args.pg_host) or valid_ip(args.pg_host)):
+        if args.pg_database:
+            db = str(args.pg_database)
+            if not valid_dbname(db):
+                raise ValueError(f"Invalid database name: {db}")
+        else:
+            db = "all"
+        pg_backup_file = backup_directory / f"{environment}-{date_prefix}-postgresql-{args.pg_host}-{db}.sql.gz"
+        if pg_backup(args, pg_backup_file):
+            result.append(pg_backup_file)
+        else:
+            all_success = False
+
     if args.arangodb_host and (valid_hostname(args.arangodb_host)):
         if args.arangodb_database:
             db = str(args.arangodb_database)
             if not valid_dbname(db):
                 raise ValueError(f"Invalid database name: {db}")
         else:
             db = "all"
-        arangodb_backup_file = backup_directory / f"{environment}-{date_prefix}-arangodb-{args.arangodb_host}-{db}.tar.gz"
+        arangodb_backup_file = (
+            backup_directory / f"{environment}-{date_prefix}-arangodb-{args.arangodb_host}-{db}.tar.gz"
+        )
         if arangodb_backup(args, arangodb_backup_file):
             result.append(arangodb_backup_file)
         else:

fixbackup/backup/postgresql.py

@@ -0,0 +1,110 @@
+import os
+import subprocess
+from pathlib import Path
+from argparse import ArgumentParser, Namespace
+from ..utils import BackupFile
+from ..logger import log
+
+
+def add_args(arg_parser: ArgumentParser) -> None:
+    arg_parser.add_argument(
+        "--pg-host",
+        help="PostgreSQL host",
+        dest="pg_host",
+        type=str,
+        default=os.getenv("PG_HOST"),
+    )
+
+    arg_parser.add_argument(
+        "--pg-port",
+        help="PostgreSQL port",
+        dest="pg_port",
+        type=int,
+        default=os.getenv("PG_PORT", 5432),
+    )
+
+    arg_parser.add_argument(
+        "--pg-user",
+        help="PostgreSQL user",
+        dest="pg_user",
+        type=str,
+        default=os.getenv("PG_USER", "postgres"),
+    )
+
+    arg_parser.add_argument(
+        "--pg-password",
+        help="PostgreSQL password",
+        dest="pg_password",
+        type=str,
+        default=os.getenv("PG_PASSWORD"),
+    )
+
+    arg_parser.add_argument(
+        "--pg-database",
+        help="PostgreSQL database",
+        dest="pg_database",
+        type=str,
+        default=os.getenv("PG_DATABASE"),
+    )
+
+    arg_parser.add_argument(
+        "--pg-dump-args",
+        help="Extra arguments to pass to pg_dump",
+        dest="pg_dump_args",
+        action="append",
+        default=[],
+    )
+
+
+def backup(args: Namespace, backup_file_path: Path, timeout: int = 900, compress: bool = True) -> bool:
+    log.info("Starting PostgreSQL backup...")
+
+    if not args.pg_host:
+        return False
+
+    env = os.environ.copy()
+    command = [
+        "pg_dump",
+        "-w",
+        "-c",
+        "--if-exists",
+        "--inserts",
+        "-h",
+        str(args.pg_host),
+        "-p",
+        str(args.pg_port),
+        "-U",
+        str(args.pg_user),
+        *args.pg_dump_args,
+    ]
+    if args.pg_database:
+        command.append("-d")
+        command.append(args.pg_database)
+    else:
+        command[0] = "pg_dumpall"
+
+    if args.pg_password:
+        env["PGPASSWORD"] = args.pg_password
+
+    log.debug(f"Running command: {' '.join(command)}")
+
+    try:
+        with BackupFile(backup_file_path, compress) as backup_fd:
+            process = subprocess.Popen(command, stdout=backup_fd, stderr=subprocess.PIPE, env=env)
+            _, stderr = process.communicate(timeout=timeout)
+
+            if process.returncode == 0:
+                log.info(f"PostgreSQL backup completed successfully. Saved to {backup_file_path}")
+                if stderr:
+                    log.debug(stderr.decode().strip())
+                return True
+            else:
+                log.error(f"PostgreSQL backup failed with return code: {process.returncode}")
+                if stderr:
+                    log.error(stderr.decode().strip())
+    except subprocess.TimeoutExpired:
+        log.error(f"PostgreSQL backup failed with timeout after {timeout} seconds")
+        process.kill()
+        process.communicate()
+
+    return False

fixbackup/logger.py

@@ -126,20 +126,15 @@ def log_to_root(message: str, *args: Any, **kwargs: Any) -> None:
 
 
 class FixLogger(Logger):
-    def debug2(self, msg: str, *args: Any, **kwargs: Any) -> None:
-        ...
+    def debug2(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
-    def debug3(self, msg: str, *args: Any, **kwargs: Any) -> None:
-        ...
+    def debug3(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
-    def debug4(self, msg: str, *args: Any, **kwargs: Any) -> None:
-        ...
+    def debug4(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
-    def debug5(self, msg: str, *args: Any, **kwargs: Any) -> None:
-        ...
+    def debug5(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
-    def trace(self, msg: str, *args: Any, **kwargs: Any) -> None:
-        ...
+    def trace(self, msg: str, *args: Any, **kwargs: Any) -> None: ...
 
 
 def get_fix_logger(name: Optional[str] = None) -> FixLogger:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fixbackup"
-version = "0.0.7"
+version = "0.0.10"
 authors = [{name="Some Engineering Inc."}]
 description = "FIX Database Backup System"
 license = {file="LICENSE"}