fix(backup,restore): make restore drills replayable and leave swarm tasks alone

Restore fixes, both hit by the infinito svc-bkp e2e drill:

- mariadb --empty dropped tables one docker exec at a time with SET
  FOREIGN_KEY_CHECKS=0 issued in a separate client session, so the
  session-scoped toggle never applied and any FK-referenced parent table
  (mailu.users) died with ERROR 1451. Issue the toggle and all DROPs in
  one session.
- postgres --empty now drops only current_user-owned objects (extension
  members like pg_trgm's set_limit are superuser-owned) with IF EXISTS
  absorbing CASCADE fallout, and the replay skips superuser-only dump
  lines (COMMENT ON EXTENSION, ALTER DEFAULT PRIVILEGES) that abort an
  app-user psql run under ON_ERROR_STOP.

Backup fixes:

- pg_dump now runs with --no-owner --no-privileges so future dumps are
  replayable by the owning app user in the first place.
- Swarm task containers are never stopped or started manually: the
  orchestrator replaces a stopped task and a later docker start fails on
  the detached overlay network. filter_stoppable skips them visibly and
  the whitelist stop check ignores them.

Validated end to end against a live infinito compose stack: the full
svc-bkp-volume-2-local drill (verify, restore cycle, sql replay for
mailu, keycloak and one more db) passes with these patches applied.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

CHANGELOG.md

@@ -1,3 +1,10 @@
+## [1.7.1] - 2026-05-26
+
+* 🔌 MariaDB SQL backups now connect over TCP loopback so the dump always matches the same wildcard-host grant the application uses — no more surprise `ERROR 1045 Access denied` when a localhost-bound auth row preempts.
+* 🧪 New regression and bug-repro tests pin the TCP behaviour and prove it under the exact preemption setup that caused the production failure on MariaDB 12.
+* 🩺 E2E test infrastructure: DinD bridge and inner daemon now default to MTU 1280 so registry pulls survive host paths with broken PMTUD (override via `E2E_DIND_MTU`).
+
+
 ## [1.7.0] - 2026-02-07
 
 * 🚀 Backup jobs now support all valid Docker Compose file names – case-insensitive and hassle-free.

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "backup-docker-to-local"
-version = "1.7.0"
+version = "1.7.1"
 description = "Backup Docker volumes to local with rsync and optional DB dumps."
 readme = "README.md"
 requires-python = ">=3.9"

scripts/test-e2e.sh

@@ -25,6 +25,8 @@ RSYNC_IMG="${E2E_RSYNC_IMAGE:-ghcr.io/kevinveenbirkenbach/alpine-rsync}"
 READY_TIMEOUT_SECONDS="${E2E_READY_TIMEOUT_SECONDS:-120}"
 ARTIFACTS_DIR="${E2E_ARTIFACTS_DIR:-./artifacts}"
 
+DIND_MTU="${E2E_DIND_MTU:-1280}"
+
 KEEP_ON_FAIL="${E2E_KEEP_ON_FAIL:-0}"
 KEEP_VOLUMES="${E2E_KEEP_VOLUMES:-0}"
 DEBUG_SHELL="${E2E_DEBUG_SHELL:-0}"
@@ -124,8 +126,11 @@ cleanup() {
 }
 trap cleanup EXIT INT TERM
 
-log "Creating network ${NET} (if missing)"
-docker network inspect "${NET}" >/dev/null 2>&1 || docker network create "${NET}" >/dev/null
+log "(Re)creating network ${NET} with MTU ${DIND_MTU}"
+docker network rm "${NET}" >/dev/null 2>&1 || true
+docker network create \
+  --opt com.docker.network.driver.mtu="${DIND_MTU}" \
+  "${NET}" >/dev/null
 
 log "Removing old ${DIND} (if any)"
 docker rm -f "${DIND}" >/dev/null 2>&1 || true
@@ -148,7 +153,8 @@ docker run -d --privileged \
   -p 2375:2375 \
   docker:dind \
   --host=tcp://0.0.0.0:2375 \
-  --tls=false >/dev/null
+  --tls=false \
+  --mtu="${DIND_MTU}" >/dev/null
 
 log "Waiting for DinD to be ready..."
 for i in $(seq 1 "${READY_TIMEOUT_SECONDS}"); do

src/baudolo/backup/app.py

@@ -16,8 +16,10 @@ from .docker import (
     change_containers_status,
     containers_using_volume,
     docker_volume_names,
+    filter_stoppable,
     get_image_info,
     has_image,
+    is_swarm_task,
 )
 from .shell import execute_shell_command
 from .volume import backup_volume
@@ -66,9 +68,13 @@ def volume_is_fully_ignored(
 
 def requires_stop(containers: list[str], images_no_stop_required: list[str]) -> bool:
     """
-    Stop is required if ANY container image is NOT in the whitelist patterns.
+    Stop is required if ANY stoppable container image is NOT in the
+    whitelist patterns. Swarm task containers never count: baudolo must
+    not cycle them (see docker.is_swarm_task).
     """
     for c in containers:
+        if is_swarm_task(c):
+            continue
         img = get_image_info(c)
         if not any(pat in img for pat in images_no_stop_required):
             return True
@@ -219,20 +225,22 @@ def main() -> int:
 
         if args.everything:
             # "everything": always do pre-rsync, then stop + rsync again
+            stoppable = filter_stoppable(containers)
             backup_volume(versions_dir, volume_name, vol_dir)
-            change_containers_status(containers, "stop")
+            change_containers_status(stoppable, "stop")
             backup_volume(versions_dir, volume_name, vol_dir)
             if not args.shutdown:
-                change_containers_status(containers, "start")
+                change_containers_status(stoppable, "start")
             continue
 
         # default: rsync, and if needed stop + rsync
         backup_volume(versions_dir, volume_name, vol_dir)
         if requires_stop(containers, args.images_no_stop_required):
-            change_containers_status(containers, "stop")
+            stoppable = filter_stoppable(containers)
+            change_containers_status(stoppable, "stop")
             backup_volume(versions_dir, volume_name, vol_dir)
             if not args.shutdown:
-                change_containers_status(containers, "start")
+                change_containers_status(stoppable, "start")
 
     # Stamp the backup version directory using dirval (python lib)
     stamp_directory(version_dir)

src/baudolo/backup/db.py

@@ -115,8 +115,10 @@ def backup_database(
         dump_file = os.path.join(out_dir, f"{db_name}.backup.sql")
 
         if db_type == "mariadb":
+            # Force TCP so auth matches '<user>'@'%' instead of socket -> 'localhost'.
             cmd = (
                 f"docker exec {container} /usr/bin/mariadb-dump "
+                f"-h 127.0.0.1 --protocol=tcp "
                 f"-u {user} -p{password} {db_name}"
             )
             _atomic_write_cmd(cmd, dump_file)
@@ -127,7 +129,8 @@ def backup_database(
             try:
                 cmd = (
                     f"PGPASSWORD={password} docker exec -i {container} "
-                    f"pg_dump -U {user} -d {db_name} -h localhost"
+                    f"pg_dump -U {user} -d {db_name} -h localhost "
+                    f"--no-owner --no-privileges"
                 )
                 _atomic_write_cmd(cmd, dump_file)
                 produced = True

src/baudolo/backup/docker.py

@@ -24,6 +24,31 @@ def containers_using_volume(volume_name: str) -> list[str]:
     )
 
 
+def is_swarm_task(container: str) -> bool:
+    """Swarm-managed task containers must never be stopped or started
+    manually: the orchestrator replaces the stopped task and a later
+    `docker start` fails on the detached overlay network."""
+    out = execute_shell_command(
+        "docker inspect --format "
+        f"'{{{{index .Config.Labels \"com.docker.swarm.task.id\"}}}}' {container}"
+    )
+    return bool(out and out[0].strip())
+
+
+def filter_stoppable(containers: list[str]) -> list[str]:
+    """Containers baudolo may stop/start itself (everything but swarm tasks)."""
+    stoppable = []
+    for container in containers:
+        if is_swarm_task(container):
+            print(
+                f"Skipping stop/start for swarm task container '{container}'.",
+                flush=True,
+            )
+            continue
+        stoppable.append(container)
+    return stoppable
+
+
 def change_containers_status(containers: list[str], status: str) -> None:
     """Stop or start a list of containers."""
     if not containers:

src/baudolo/restore/db/mariadb.py

@@ -47,18 +47,6 @@ def restore_mariadb_sql(
         # IMPORTANT:
         # Do NOT hardcode 'mysql' here. Use the detected client.
         # MariaDB 11 images may not contain the mysql binary at all.
-        docker_exec(
-            container,
-            [
-                client,
-                "-u",
-                user,
-                f"--password={password}",
-                "-e",
-                "SET FOREIGN_KEY_CHECKS=0;",
-            ],
-        )
-
         result = docker_exec(
             container,
             [
@@ -74,7 +62,16 @@ def restore_mariadb_sql(
         )
         tables = result.stdout.decode().split()
 
-        for tbl in tables:
+        if tables:
+            # SET FOREIGN_KEY_CHECKS is session-scoped, so it must share one
+            # client session with the DROPs or FK constraints still fire.
+            drop_sql = (
+                "SET FOREIGN_KEY_CHECKS=0; "
+                + " ".join(
+                    f"DROP TABLE IF EXISTS `{db_name}`.`{tbl}`;" for tbl in tables
+                )
+                + " SET FOREIGN_KEY_CHECKS=1;"
+            )
             docker_exec(
                 container,
                 [
@@ -83,22 +80,10 @@ def restore_mariadb_sql(
                     user,
                     f"--password={password}",
                     "-e",
-                    f"DROP TABLE IF EXISTS `{db_name}`.`{tbl}`;",
+                    drop_sql,
                 ],
             )
 
-        docker_exec(
-            container,
-            [
-                client,
-                "-u",
-                user,
-                f"--password={password}",
-                "-e",
-                "SET FOREIGN_KEY_CHECKS=1;",
-            ],
-        )
-
     with open(sql_path, "rb") as f:
         docker_exec(
             container, [client, "-u", user, f"--password={password}", db_name], stdin=f

src/baudolo/restore/db/postgres.py

@@ -21,17 +21,43 @@ def restore_postgres_sql(
     docker_env = {"PGPASSWORD": password}
 
     if empty:
+        # Owner-filtered: extension members (pg_trgm's set_limit) are superuser-owned; IF EXISTS absorbs CASCADE fallout.
         drop_sql = r"""
 DO $$ DECLARE r RECORD;
 BEGIN
   FOR r IN (
-    SELECT table_name AS name, 'TABLE' AS type FROM information_schema.tables WHERE table_schema='public'
+    SELECT c.relname AS name,
+           CASE c.relkind
+             WHEN 'v' THEN 'VIEW'
+             WHEN 'm' THEN 'MATERIALIZED VIEW'
+             WHEN 'f' THEN 'FOREIGN TABLE'
+             ELSE 'TABLE'
+           END AS type
+      FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+     WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p', 'v', 'm', 'f')
+       AND pg_get_userbyid(c.relowner) = current_user
     UNION ALL
-    SELECT routine_name AS name, 'FUNCTION' AS type FROM information_schema.routines WHERE specific_schema='public'
+    SELECT p.proname AS name,
+           CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END AS type
+      FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace
+     WHERE n.nspname = 'public' AND p.prokind IN ('f', 'p', 'w')
+       AND pg_get_userbyid(p.proowner) = current_user
     UNION ALL
-    SELECT sequence_name AS name, 'SEQUENCE' AS type FROM information_schema.sequences WHERE sequence_schema='public'
+    SELECT c.relname AS name, 'SEQUENCE' AS type
+      FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+     WHERE n.nspname = 'public' AND c.relkind = 'S'
+       AND pg_get_userbyid(c.relowner) = current_user
+    UNION ALL
+    SELECT t.typname AS name, 'TYPE' AS type
+      FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace
+     WHERE n.nspname = 'public'
+       AND pg_get_userbyid(t.typowner) = current_user
+       AND (t.typtype IN ('e', 'd')
+            OR (t.typtype = 'c' AND EXISTS (
+                  SELECT 1 FROM pg_class c2
+                   WHERE c2.oid = t.typrelid AND c2.relkind = 'c')))
   ) LOOP
-    EXECUTE format('DROP %s public.%I CASCADE', r.type, r.name);
+    EXECUTE format('DROP %s IF EXISTS public.%I CASCADE', r.type, r.name);
   END LOOP;
 END $$;
 """
@@ -43,11 +69,18 @@ END $$;
         )
 
     with open(sql_path, "rb") as f:
-        docker_exec(
-            container,
-            ["psql", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", db_name],
-            stdin=f,
-            docker_env=docker_env,
-        )
+        raw_sql = f.read()
+    # COMMENT ON EXTENSION and ALTER DEFAULT PRIVILEGES are superuser-only;
+    # app-level restores must skip them or ON_ERROR_STOP aborts the replay.
+    superuser_only = (b"COMMENT ON EXTENSION", b"ALTER DEFAULT PRIVILEGES")
+    sql = b"\n".join(
+        line for line in raw_sql.splitlines() if not line.startswith(superuser_only)
+    )
+    docker_exec(
+        container,
+        ["psql", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", db_name],
+        stdin=sql,
+        docker_env=docker_env,
+    )
 
     print(f"PostgreSQL restore complete for db '{db_name}'.")

tests/e2e/test_e2e_mariadb_anonymous_preemption.py

@@ -0,0 +1,162 @@
+"""
+Bug-repro for: mariadb-dump fails with `ERROR 1045 Access denied for user
+'<u>'@'localhost' (using password: YES)` when only '<u>'@'%' is granted and a
+preempting ''@'localhost' user is present.
+
+The fix forces TCP loopback in baudolo.backup.db so the dump matches the
+'<u>'@'%' grant instead of the socket->localhost auth row.
+
+This file:
+- builds the exact preconditions that triggered the production failure,
+- as a NEGATIVE control, runs a socket-based mariadb-dump (== the old code path)
+  and asserts that it fails with the literal 1045 / @'localhost' error,
+- as a POSITIVE proof, calls backup_database() (where the fix lives) against
+  the same DB container and asserts the dump file is produced and contains the
+  seed data.
+
+Note: the volume-rsync stage of baudolo is intentionally NOT exercised here.
+That stage needs root on /var/lib/docker/volumes, which is provided by the
+DinD wrapper in `make test-e2e` but not by an on-host invocation. The bug we
+are verifying is in the DB-dump stage, so testing backup_database() directly
+keeps the assertion focused and the test runnable both on-host and in DinD.
+"""
+
+import os
+import tempfile
+import unittest
+
+import pandas
+
+from baudolo.backup import db as db_mod
+
+from .helpers import (
+    cleanup_docker,
+    require_docker,
+    run,
+    unique,
+    wait_for_mariadb,
+    wait_for_mariadb_sql,
+)
+
+
+class TestE2EMariaDBAnonymousPreemption(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        require_docker()
+        cls.prefix = unique("baudolo-e2e-mariadb-anon")
+        cls.db_container = f"{cls.prefix}-mariadb"
+        cls.db_volume = f"{cls.prefix}-mariadb-vol"
+        cls.containers = [cls.db_container]
+        cls.volumes = [cls.db_volume]
+
+        cls.db_name = "appdb"
+        cls.db_user = "tcponly"
+        cls.db_password = "tcponlypw"
+        cls.root_password = "rootpw"
+
+        run(["docker", "volume", "create", cls.db_volume])
+
+        # Boot WITHOUT MARIADB_USER/MARIADB_PASSWORD/MARIADB_DATABASE so the
+        # entrypoint does not auto-create '<u>'@'%'. We provision the user
+        # explicitly below to mirror the SQL path used by svc-db-mariadb.
+        run(
+            [
+                "docker",
+                "run",
+                "-d",
+                "--name",
+                cls.db_container,
+                "-e",
+                f"MARIADB_ROOT_PASSWORD={cls.root_password}",
+                "-v",
+                f"{cls.db_volume}:/var/lib/mysql",
+                "mariadb:12.2",
+            ]
+        )
+
+        wait_for_mariadb(
+            cls.db_container, root_password=cls.root_password, timeout_s=120
+        )
+
+        # Provision: '<u>'@'%' (the app/backup grant) + anonymous ''@'localhost'
+        # (the preemption trigger). Mirrors the production state that produced
+        # `ERROR 1045 ... '<u>'@'localhost' (using password: YES)`.
+        bootstrap_sql = (
+            f"CREATE DATABASE {cls.db_name};"
+            f"CREATE USER '{cls.db_user}'@'%' IDENTIFIED BY '{cls.db_password}';"
+            f"GRANT ALL PRIVILEGES ON {cls.db_name}.* TO '{cls.db_user}'@'%';"
+            f"CREATE USER ''@'localhost' IDENTIFIED BY 'anonpw-not-{cls.db_password}';"
+            "FLUSH PRIVILEGES;"
+            f"CREATE TABLE {cls.db_name}.t (id INT PRIMARY KEY, v VARCHAR(50));"
+            f"INSERT INTO {cls.db_name}.t VALUES (1,'ok');"
+        )
+        run(
+            [
+                "docker",
+                "exec",
+                cls.db_container,
+                "sh",
+                "-lc",
+                f'mariadb -uroot --protocol=socket -e "{bootstrap_sql}"',
+            ]
+        )
+
+        # Sanity: '<u>' can log in over TCP (matches '%'). If THIS fails,
+        # the precondition for the fix to even apply is broken.
+        wait_for_mariadb_sql(
+            cls.db_container, user=cls.db_user, password=cls.db_password, timeout_s=60
+        )
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cleanup_docker(containers=cls.containers, volumes=cls.volumes)
+
+    def test_negative_control_socket_dump_fails_with_1045(self) -> None:
+        # Reproduces the OLD code path (no -h/--protocol). MUST fail with 1045
+        # under the configured preemption. If this ever starts passing, either
+        # the MariaDB auth semantics changed or the anonymous-user setup did
+        # not take effect — in both cases the positive test below loses its
+        # ability to discriminate "fix works" vs "bug never reproduced".
+        p = run(
+            [
+                "docker",
+                "exec",
+                self.db_container,
+                "sh",
+                "-lc",
+                f"mariadb-dump -u{self.db_user} -p{self.db_password} {self.db_name}",
+            ],
+            capture=True,
+            check=False,
+        )
+        self.assertNotEqual(p.returncode, 0, "socket-based dump unexpectedly succeeded")
+        self.assertIn("1045", (p.stderr or "") + (p.stdout or ""))
+        self.assertIn("@'localhost'", (p.stderr or "") + (p.stdout or ""))
+
+    def test_backup_database_succeeds_with_tcp_fix(self) -> None:
+        # Drives the function where the fix lives. No rsync, no privileged
+        # paths — just the dump that the negative-control proved is failing
+        # under the same preemption setup.
+        with tempfile.TemporaryDirectory() as volume_dir:
+            df = pandas.DataFrame(
+                [(self.db_container, self.db_name, self.db_user, self.db_password)],
+                columns=["instance", "database", "username", "password"],
+            )
+            produced = db_mod.backup_database(
+                container=self.db_container,
+                volume_dir=volume_dir,
+                db_type="mariadb",
+                databases_df=df,
+                database_containers=[self.db_container],
+            )
+            self.assertTrue(produced, "backup_database did not produce a dump")
+            dump_path = os.path.join(volume_dir, "sql", f"{self.db_name}.backup.sql")
+            self.assertTrue(os.path.isfile(dump_path), f"expected dump at {dump_path}")
+            with open(dump_path, "r", encoding="utf-8", errors="replace") as f:
+                content = f.read()
+            self.assertIn("INSERT INTO", content)
+            self.assertIn("'ok'", content)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

tests/unit/backup/test_db_mariadb_dump.py

@@ -0,0 +1,70 @@
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import pandas
+
+from baudolo.backup import db as db_mod
+
+
+def _df(rows):
+    return pandas.DataFrame(
+        rows, columns=["instance", "database", "username", "password"]
+    )
+
+
+def _capture_commands(*, db_type, rows, container):
+    captured = []
+
+    def _capture(cmd):
+        captured.append(cmd)
+        return []
+
+    with tempfile.TemporaryDirectory() as td:
+        with patch.object(db_mod, "execute_shell_command", side_effect=_capture):
+            db_mod.backup_database(
+                container=container,
+                volume_dir=td,
+                db_type=db_type,
+                databases_df=_df(rows),
+                database_containers=[container],
+            )
+    return captured
+
+
+class TestMariaDBDumpUsesTCP(unittest.TestCase):
+    # Regression guard for 'Access denied for user <user>@localhost' when only
+    # '<user>'@'%' is granted: the in-container mariadb-dump MUST force TCP so
+    # the connection is auth-matched against '%' instead of socket->localhost.
+
+    def test_mariadb_dump_forces_tcp_loopback(self):
+        captured = _capture_commands(
+            db_type="mariadb",
+            rows=[("mariadb", "appdb", "appuser", "s3cret")],
+            container="mariadb",
+        )
+        dump_cmds = [c for c in captured if "mariadb-dump" in c]
+        self.assertEqual(
+            len(dump_cmds), 1, f"expected one dump command, got: {captured}"
+        )
+
+        cmd = dump_cmds[0]
+        self.assertIn("-h 127.0.0.1", cmd)
+        self.assertIn("--protocol=tcp", cmd)
+        self.assertIn("-u appuser", cmd)
+        self.assertIn("-ps3cret", cmd)
+        self.assertIn(" appdb", cmd)
+
+    def test_postgres_dump_unaffected(self):
+        captured = _capture_commands(
+            db_type="postgres",
+            rows=[("pg", "appdb", "appuser", "s3cret")],
+            container="pg",
+        )
+        dump_cmds = [c for c in captured if "pg_dump" in c and "pg_dumpall" not in c]
+        self.assertEqual(len(dump_cmds), 1)
+        self.assertNotIn("--protocol=tcp", dump_cmds[0])
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)