feat(backup): write database dumps gzip-compressed

Dumps stream through gzip into <db>.backup.sql.gz; no uncompressed copy touches the disk. Restore reads gzip and plain dumps by their magic bytes, verifies a gzip dump to its end before anything is dropped, and still finds the uncompressed names of older generations. The manifest names the compression.

🎯 Fix: 92 %
💥 Break: 20 %
💸 Cost: –
⏱️ Time: –
🛡️ Security: ~100 %
🏆 Score: 74
🚦 Gate: ⛔ break 20 % > 5

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/baudolo/backup/db.py

@@ -9,7 +9,7 @@ from baudolo.databases import CLUSTER_ROW, validate_database
 from baudolo.generation import CLUSTER_SUFFIX, DUMP_SUFFIX, SQL_DIR
 
 from .docker import docker_exec_argv
-from .shell import BackupError, execute_to_file
+from .shell import BackupError, execute_to_gzip
 
 if TYPE_CHECKING:
     import pandas as pd
@@ -53,7 +53,7 @@ def fallback_pg_dumpall(
     """
     Perform a full Postgres cluster dump using pg_dumpall.
     """
-    execute_to_file(
+    execute_to_gzip(
         docker_exec_argv(
             container,
             ["pg_dumpall", "-U", username, "-h", "localhost"],
@@ -122,7 +122,7 @@ def backup_database(
 
         if db_type == "mariadb":
             # Force TCP so auth matches '<user>'@'%' instead of socket -> 'localhost'.
-            execute_to_file(
+            execute_to_gzip(
                 docker_exec_argv(
                     container,
                     [
@@ -144,7 +144,7 @@ def backup_database(
 
         if db_type == "postgres":
             try:
-                execute_to_file(
+                execute_to_gzip(
                     docker_exec_argv(
                         container,
                         [

src/baudolo/backup/shell.py

@@ -7,14 +7,22 @@ string handed to ``shell=True`` allowed.
 
 from __future__ import annotations
 
+import gzip
 import os
 import subprocess
+import tempfile
+import zlib
+from functools import partial
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from collections.abc import Mapping, Sequence
 
+GZIP_LEVEL = 6
+READ_BYTES = 1 << 20
+SYNC_BYTES = 1 << 16
+
 
 class BackupError(Exception):
     """Generic exception for backup errors."""
@@ -53,23 +61,71 @@ def execute_shell_command(
     return [line.decode("utf-8") for line in out.splitlines()]
 
 
-def execute_to_file(
+def is_sync_point(piece: bytes) -> bool:
+    """
+    Args:
+        piece: a line of the dump, or a READ_BYTES slice of a longer one.
+
+    Returns:
+        True when a full flush follows *piece*, decided by its bytes alone, on
+        average once per SYNC_BYTES of input.
+    """
+    return zlib.crc32(piece) % max(1, SYNC_BYTES // len(piece)) == 0
+
+
+def _abort(process: subprocess.Popen, tmp: Path) -> None:
+    process.kill()
+    process.wait()
+    tmp.unlink(missing_ok=True)
+
+
+def execute_to_gzip(
     command: Sequence[str], out_file: str, *, env: Mapping[str, str] | None = None
 ) -> None:
-    """Run *command*, writing its stdout to *out_file* only once it succeeded.
+    """
+    Args:
+        command: argv, the program first; its stdout is the dump.
+        out_file: the final gzip path. The stream is written to
+            ``<out_file>.tmp`` and renamed only after the command exited 0.
+        env: variables added to the child's environment.
 
-    The output goes to a sibling temporary file first, so a partial or empty
-    stream from a failing dump never takes the place of a valid backup.
+    Raises:
+        BackupError: the command failed, or the file could not be written.
     """
     command = list(command)
     print(" ".join(command), flush=True)
     tmp = Path(f"{out_file}.tmp")
-    with tmp.open("wb") as handle:
+    with tempfile.TemporaryFile() as err_sink:
         process = subprocess.Popen(
-            command, stdout=handle, stderr=subprocess.PIPE, env=_child_env(env)
+            command, stdout=subprocess.PIPE, stderr=err_sink, env=_child_env(env)
         )
-        _, err = process.communicate()
-    if process.returncode != 0:
+        try:
+            with (
+                tmp.open("wb") as raw,
+                gzip.GzipFile(
+                    filename="",
+                    mode="wb",
+                    fileobj=raw,
+                    compresslevel=GZIP_LEVEL,
+                    mtime=0,
+                ) as packed,
+            ):
+                for piece in iter(partial(process.stdout.readline, READ_BYTES), b""):
+                    packed.write(piece)
+                    if is_sync_point(piece):
+                        packed.flush(zlib.Z_FULL_FLUSH)
+        except OSError as error:
+            _abort(process, tmp)
+            raise BackupError(f"Error writing {out_file}: {error}") from error
+        except BaseException:
+            _abort(process, tmp)
+            raise
+        finally:
+            process.stdout.close()
+        returncode = process.wait()
+        err_sink.seek(0)
+        err = err_sink.read()
+    if returncode != 0:
         tmp.unlink()
-        _fail(command, process.returncode, b"", err)
+        _fail(command, returncode, b"", err)
     tmp.replace(out_file)