Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d672388f63 | ||
|
|
5817078643 | ||
|
|
4214d3709a | ||
|
|
4c51da009c | ||
|
|
b43177f0e7 | ||
|
|
0735e5fa7b | ||
|
|
389b418f42 | ||
|
|
948b8347f4 | ||
|
|
f824fabea1 | ||
|
|
07d819cb41 |
No files matched your search
@@ -1,5 +1,59 @@
|
||||
# Changelog
|
||||
|
||||
## [7.2.0] - 2026-11-03
|
||||
|
||||
Added:
|
||||
|
||||
- Seed: *--password-stdin* reads the database password from stdin
|
||||
- Seed: a password given both ways or not at all is rejected
|
||||
|
||||
Changed:
|
||||
|
||||
- Seed: the password argument still works but warns it is visible in *ps*
|
||||
|
||||
Upgrade notes:
|
||||
|
||||
- Seed: pass the password via *--password-stdin*; the argument is deprecated
|
||||
|
||||
## [7.1.2] - 2026-10-18
|
||||
|
||||
Fixed:
|
||||
|
||||
- Backup: *--snapshot zfs* found no snapshot inside a container; now mounts it
|
||||
- Tests: e2e zfs pool now sits on a loop device; a file vdev never worked
|
||||
|
||||
Upgrade notes:
|
||||
|
||||
- Backup: zfs snapshots mount at *.baudolo-TAG* in the subject, not *.zfs*
|
||||
- Image: first ghcr image since 3.1.2; the 4.0.0 to 7.1.1 notes apply
|
||||
|
||||
## [7.1.1] - 2026-10-17
|
||||
|
||||
Fixed:
|
||||
|
||||
- Tests: e2e MariaDB probe ran root passwordless and timed out at random
|
||||
- Tests: probe now logs in with the root password, skips the temp server
|
||||
- Tests: anonymous-preemption bootstrap now uses the root password too
|
||||
|
||||
Upgrade notes:
|
||||
|
||||
- 7.1.0 never reached stable: its CI failed, no image was published
|
||||
- 7.1.1 is the first published -aHAX build; 7.1.0 upgrade notes apply
|
||||
|
||||
## [7.1.0] - 2026-10-17
|
||||
|
||||
Fixed:
|
||||
|
||||
- Backup: rsync -a stored no xattrs, ACLs or hardlinks; now -aHAX
|
||||
- Restore: restore_volume_files had the same gap; now -aHAX too
|
||||
- An OpenCloud volume lost its storage metadata through a DR drill
|
||||
|
||||
Upgrade notes:
|
||||
|
||||
- First backup re-copies every attribute-carrying volume once in full
|
||||
- A destination without xattr support now fails the backup run outright
|
||||
- exFAT, xattr-less NFS and object-storage FUSE mounts must move first
|
||||
|
||||
## [7.0.2] - 2026-09-23
|
||||
|
||||
* Backup: failed live pre-copy no longer aborts when a stopped copy follows
|
||||
|
||||
+2
-2
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "backup-docker-to-local"
|
||||
version = "7.0.2"
|
||||
version = "7.2.0"
|
||||
description = "Backup Docker volumes to local with rsync and optional DB dumps."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
@@ -19,7 +19,7 @@ dependencies = [
|
||||
[project.optional-dependencies]
|
||||
# Pinned: a ruff minor bump changes which rules fire, and `make test` gates on
|
||||
# a clean run, so an unpinned lint would fail the suite on an unrelated day.
|
||||
lint = ["ruff==0.16.7"]
|
||||
lint = ["ruff==0.17.3"]
|
||||
|
||||
[project.scripts]
|
||||
baudolo = "baudolo.backup.__main__:main"
|
||||
|
||||
@@ -58,27 +58,50 @@ def _resolver(subject: str, root: str) -> Callable[[str], str]:
|
||||
return resolve
|
||||
|
||||
|
||||
def _remove(
|
||||
root: str, commands: list[list[str]], run: Callable[[list[str]], list[str]]
|
||||
) -> None:
|
||||
try:
|
||||
for command in commands:
|
||||
run(command)
|
||||
except BackupError as error:
|
||||
# Raising here would also mask whatever the body raised.
|
||||
print(f"WARNING: {root} could not be removed: {error}", flush=True)
|
||||
|
||||
|
||||
def _btrfs(
|
||||
subject: str, name: str, run: Callable[[list[str]], list[str]]
|
||||
) -> tuple[str, list[str]]:
|
||||
) -> tuple[str, list[list[str]]]:
|
||||
# The snapshot goes inside the subject, never beside it: the kernel rejects
|
||||
# a snapshot whose destination is on another filesystem, which is exactly
|
||||
# what the parent directory is when the subject is a mountpoint of its own.
|
||||
target = str(Path(os.path.abspath(subject)) / f".{name}") # noqa: PTH100 - see _resolver
|
||||
run(["btrfs", "subvolume", "snapshot", "-r", subject, target])
|
||||
return target, ["btrfs", "subvolume", "delete", target]
|
||||
return target, [["btrfs", "subvolume", "delete", target]]
|
||||
|
||||
|
||||
def _zfs(
|
||||
subject: str, name: str, run: Callable[[list[str]], list[str]]
|
||||
) -> tuple[str, list[str]]:
|
||||
) -> tuple[str, list[list[str]]]:
|
||||
output = run(["zfs", "list", "-H", "-o", "name", subject])
|
||||
dataset = (output[0] if output else "").strip()
|
||||
if not dataset:
|
||||
raise SnapshotError(f"no zfs dataset is mounted at {subject}")
|
||||
run(["zfs", "snapshot", f"{dataset}@{name}"])
|
||||
root = str(Path(subject) / ".zfs" / "snapshot" / name)
|
||||
return root, ["zfs", "destroy", f"{dataset}@{name}"]
|
||||
snapshot = f"{dataset}@{name}"
|
||||
# Exception: mounted here rather than read through .zfs/snapshot, which the
|
||||
# kernel automounts via a usermode helper in the host's mount namespace,
|
||||
# so a containerised run never sees it.
|
||||
root = str(Path(os.path.abspath(subject)) / f".{name}") # noqa: PTH100 - see _resolver
|
||||
run(["zfs", "snapshot", snapshot])
|
||||
remove = [["zfs", "destroy", snapshot]]
|
||||
try:
|
||||
run(["mkdir", root])
|
||||
remove.insert(0, ["rmdir", root])
|
||||
run(["mount", "-t", "zfs", snapshot, root])
|
||||
except BaseException:
|
||||
_remove(root, remove, run)
|
||||
raise
|
||||
return root, [["umount", root], *remove]
|
||||
|
||||
|
||||
_CREATE = {"btrfs": _btrfs, "zfs": _zfs}
|
||||
@@ -169,8 +192,4 @@ def volume_snapshot(
|
||||
try:
|
||||
yield _resolver(subject, root)
|
||||
finally:
|
||||
try:
|
||||
run(remove)
|
||||
except BackupError as error:
|
||||
# Raising here would also mask whatever the body raised.
|
||||
print(f"WARNING: {root} could not be removed: {error}", flush=True)
|
||||
_remove(root, remove, run)
|
||||
@@ -75,7 +75,7 @@ def backup_volume(
|
||||
pathlib.Path(dest).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
last = get_last_backup_dir(versions_dir, volume_name, dest)
|
||||
cmd = ["rsync", "-aP", "--no-D", "--delete", "--delete-excluded"]
|
||||
cmd = ["rsync", "-aHAX", "-P", "--no-D", "--delete", "--delete-excluded"]
|
||||
if authoritative:
|
||||
cmd.append("--checksum")
|
||||
if last:
|
||||
|
||||
@@ -59,6 +59,6 @@ def restore_volume_files(volume_name: str, backup_files_dir: str) -> int:
|
||||
# rsync reads "dir/" as its contents and "dir" as the directory itself.
|
||||
src = f"{Path(backup_files_dir)}{os.sep}"
|
||||
dest = f"{Path(mountpoint)}{os.sep}"
|
||||
run(["rsync", "-avv", "--delete", src, dest])
|
||||
run(["rsync", "-aHAX", "-vv", "--delete", src, dest])
|
||||
print("File restore complete.")
|
||||
return 0
|
||||
@@ -73,9 +73,30 @@ def main() -> None:
|
||||
help="Database name or '*' to dump all databases",
|
||||
)
|
||||
parser.add_argument("username", help="Database username")
|
||||
parser.add_argument("password", help="Database password")
|
||||
parser.add_argument(
|
||||
"password",
|
||||
nargs="?",
|
||||
help="Deprecated: database password as an argument, visible in the "
|
||||
"process list; use --password-stdin",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--password-stdin",
|
||||
action="store_true",
|
||||
help="Read the database password from stdin",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.password_stdin == (args.password is not None):
|
||||
parser.error("give the password either via --password-stdin or as an argument")
|
||||
if args.password_stdin:
|
||||
password = sys.stdin.read().removesuffix("\n")
|
||||
else:
|
||||
print(
|
||||
"WARNING: a password argument is visible in the process list; "
|
||||
"use --password-stdin",
|
||||
file=sys.stderr,
|
||||
)
|
||||
password = args.password
|
||||
|
||||
try:
|
||||
check_and_add_entry(
|
||||
@@ -83,7 +104,7 @@ def main() -> None:
|
||||
instance=args.instance,
|
||||
database=args.database,
|
||||
username=args.username,
|
||||
password=args.password,
|
||||
password=password,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - CLI boundary: any failure becomes exit 1
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
|
||||
@@ -96,12 +96,13 @@ def wait_for_mariadb(
|
||||
container: str, *, root_password: str, timeout_s: int = 90
|
||||
) -> None:
|
||||
"""
|
||||
Liveness probe for MariaDB.
|
||||
Readiness probe for MariaDB, as root over the socket.
|
||||
|
||||
IMPORTANT (MariaDB 11):
|
||||
Root TCP auth is often restricted (unix_socket auth), so a TCP ping like
|
||||
`mariadb-admin -uroot -p... -h localhost ping` can fail even though the server is up.
|
||||
We therefore check readiness via a socket-based query.
|
||||
The entrypoint drops root@127.0.0.1, so root is socket-only, and it
|
||||
installs root@localhost without unix_socket auth: a probe without the
|
||||
password only passes in the init window before the password is set.
|
||||
`@@skip_networking` must be 0 so the entrypoint's temporary server,
|
||||
which shuts down right after init, does not count as ready.
|
||||
"""
|
||||
deadline = time.time() + timeout_s
|
||||
while time.time() < deadline:
|
||||
@@ -112,12 +113,15 @@ def wait_for_mariadb(
|
||||
container,
|
||||
"sh",
|
||||
"-lc",
|
||||
'mariadb -uroot --protocol=socket -e "SELECT 1;"',
|
||||
(
|
||||
f"mariadb -uroot -p{root_password} --protocol=socket "
|
||||
'-N -e "SELECT @@skip_networking;"'
|
||||
),
|
||||
],
|
||||
capture=True,
|
||||
check=False,
|
||||
)
|
||||
if p.returncode == 0:
|
||||
if p.returncode == 0 and p.stdout.strip() == "0":
|
||||
return
|
||||
time.sleep(1)
|
||||
raise TimeoutError(
|
||||
|
||||
@@ -100,7 +100,10 @@ class TestE2EMariaDBAnonymousPreemption(unittest.TestCase):
|
||||
cls.db_container,
|
||||
"sh",
|
||||
"-lc",
|
||||
f'mariadb -uroot --protocol=socket -e "{bootstrap_sql}"',
|
||||
(
|
||||
f"mariadb -uroot -p{cls.root_password} --protocol=socket "
|
||||
f'-e "{bootstrap_sql}"'
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -36,9 +36,14 @@ ZFS_READY = "{ [ -c /dev/zfs ] || modprobe zfs 2>/dev/null; }; [ -c /dev/zfs ]"
|
||||
def mount_script(fstype: str) -> str:
|
||||
"""Build a filesystem on a loop device and carve out the snapshot subject."""
|
||||
if fstype == "zfs":
|
||||
# Exception: a loop device, not the /img file, because the kernel opens
|
||||
# vdevs from a taskq thread that resolves paths in the host's root.
|
||||
# The pool is kernel-wide, so it is destroyed on exit, but only once
|
||||
# this script created it.
|
||||
return (
|
||||
f"{PACKAGES} && {ZFS_READY} && truncate -s 400M /img "
|
||||
"&& zpool create -m none baudolo /img "
|
||||
f"{PACKAGES} && {ZFS_READY} && truncate -s 400M /img && {ATTACH} "
|
||||
'&& zpool create -m none baudolo "$LOOP" '
|
||||
"&& trap 'zpool destroy -f baudolo; losetup -d \"$LOOP\"' EXIT "
|
||||
"&& zfs create -o mountpoint=/subject/docker baudolo/docker"
|
||||
)
|
||||
return (
|
||||
|
||||
@@ -65,6 +65,32 @@ def run_seed_expect_fail(
|
||||
)
|
||||
|
||||
|
||||
def run_seed_stdin(
|
||||
csv_path: Path, instance: str, database: str, username: str, stdin: str, *extra: str
|
||||
) -> subprocess.CompletedProcess:
|
||||
"""
|
||||
Run the real CLI with --password-stdin, feeding the password on stdin.
|
||||
Any extra arguments are appended after the username.
|
||||
"""
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"baudolo.seed",
|
||||
"--password-stdin",
|
||||
str(csv_path),
|
||||
instance,
|
||||
database,
|
||||
username,
|
||||
*extra,
|
||||
],
|
||||
input=stdin,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def read_csv_semicolon(path: Path) -> list[dict]:
|
||||
with path.open("r", encoding="utf-8", newline="") as f:
|
||||
reader = csv.DictReader(f, delimiter=";")
|
||||
@@ -195,6 +221,61 @@ class TestSeedIntegration(unittest.TestCase):
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["database"], "my_db-1")
|
||||
|
||||
def test_password_stdin_stores_the_password_without_its_trailing_newline(
|
||||
self,
|
||||
) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td) / "databases.csv"
|
||||
|
||||
cp = run_seed_stdin(p, "docker.test", "appdb", "alice", "s3cr;et\n")
|
||||
|
||||
self.assertEqual(cp.returncode, 0, cp.stderr)
|
||||
self.assertNotIn("WARNING", cp.stderr)
|
||||
rows = read_csv_semicolon(p)
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["username"], "alice")
|
||||
self.assertEqual(rows[0]["password"], "s3cr;et")
|
||||
|
||||
def test_password_argument_still_works_with_a_deprecation_warning(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td) / "databases.csv"
|
||||
|
||||
cp = run_seed(p, "docker.test", "appdb", "alice", "secret")
|
||||
|
||||
self.assertIn("--password-stdin", cp.stderr)
|
||||
self.assertEqual(read_csv_semicolon(p)[0]["password"], "secret")
|
||||
|
||||
def test_password_given_both_ways_is_rejected(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td) / "databases.csv"
|
||||
|
||||
cp = run_seed_stdin(p, "docker.test", "appdb", "alice", "pw\n", "pw")
|
||||
|
||||
self.assertNotEqual(cp.returncode, 0)
|
||||
self.assertFalse(p.exists(), "Should not create file on invalid input")
|
||||
|
||||
def test_password_given_no_way_is_rejected(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td) / "databases.csv"
|
||||
|
||||
cp = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"baudolo.seed",
|
||||
str(p),
|
||||
"docker.test",
|
||||
"appdb",
|
||||
"alice",
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
self.assertNotEqual(cp.returncode, 0)
|
||||
self.assertFalse(p.exists(), "Should not create file on invalid input")
|
||||
|
||||
def test_file_is_semicolon_delimited_and_has_header(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
p = Path(td) / "databases.csv"
|
||||
|
||||
@@ -21,6 +21,14 @@ class Runner:
|
||||
return []
|
||||
|
||||
|
||||
class Unmountable(Runner):
|
||||
def __call__(self, command: list[str]) -> list[str]:
|
||||
result = super().__call__(command)
|
||||
if command[0] == "mount":
|
||||
raise BackupError("permission denied")
|
||||
return result
|
||||
|
||||
|
||||
class TestBtrfs(unittest.TestCase):
|
||||
def test_it_creates_a_read_only_snapshot_inside_the_subject(self) -> None:
|
||||
run = Runner()
|
||||
@@ -95,14 +103,57 @@ class TestZfs(unittest.TestCase):
|
||||
run.calls[-1], ["zfs", "destroy", "tank/docker@baudolo-20260731"]
|
||||
)
|
||||
|
||||
def test_it_maps_a_volume_path_through_the_dot_zfs_directory(self) -> None:
|
||||
def test_it_mounts_the_snapshot_inside_the_subject(self) -> None:
|
||||
run = self._run()
|
||||
with volume_snapshot("zfs", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
self.assertIn(
|
||||
[
|
||||
"mount",
|
||||
"-t",
|
||||
"zfs",
|
||||
"tank/docker@baudolo-20260731",
|
||||
"/var/lib/docker/.baudolo-20260731",
|
||||
],
|
||||
run.calls,
|
||||
)
|
||||
|
||||
def test_it_maps_a_volume_path_into_the_mounted_snapshot(self) -> None:
|
||||
run = self._run()
|
||||
with volume_snapshot("zfs", "/var/lib/docker", "20260731", run=run) as resolve:
|
||||
self.assertEqual(
|
||||
resolve("/var/lib/docker/volumes/postgres_data/_data"),
|
||||
"/var/lib/docker/.zfs/snapshot/baudolo-20260731/volumes/postgres_data/_data",
|
||||
"/var/lib/docker/.baudolo-20260731/volumes/postgres_data/_data",
|
||||
)
|
||||
|
||||
def test_it_unmounts_before_it_destroys(self) -> None:
|
||||
run = self._run()
|
||||
with volume_snapshot("zfs", "/var/lib/docker", "20260731", run=run):
|
||||
pass
|
||||
self.assertEqual(
|
||||
run.calls[-3:],
|
||||
[
|
||||
["umount", "/var/lib/docker/.baudolo-20260731"],
|
||||
["rmdir", "/var/lib/docker/.baudolo-20260731"],
|
||||
["zfs", "destroy", "tank/docker@baudolo-20260731"],
|
||||
],
|
||||
)
|
||||
|
||||
def test_a_failed_mount_leaves_no_snapshot_behind(self) -> None:
|
||||
run = Unmountable({"zfs list": ["tank/docker"]})
|
||||
with (
|
||||
self.assertRaises(BackupError),
|
||||
volume_snapshot("zfs", "/var/lib/docker", "20260731", run=run),
|
||||
):
|
||||
pass
|
||||
self.assertEqual(
|
||||
run.calls[-2:],
|
||||
[
|
||||
["rmdir", "/var/lib/docker/.baudolo-20260731"],
|
||||
["zfs", "destroy", "tank/docker@baudolo-20260731"],
|
||||
],
|
||||
)
|
||||
|
||||
def test_an_unmounted_dataset_is_an_error(self) -> None:
|
||||
run = Runner({"zfs list": [""]})
|
||||
with (
|
||||
|
||||
@@ -46,9 +46,12 @@ class TestBackupVolume(unittest.TestCase):
|
||||
def test_it_carries_no_kernel_objects_into_a_generation(self) -> None:
|
||||
self.assertIn("--no-D", self.copy())
|
||||
|
||||
def test_it_stores_the_metadata_a_restore_has_to_give_back(self) -> None:
|
||||
self.assertIn("-aHAX", self.copy())
|
||||
|
||||
def test_it_keeps_no_twin_of_what_the_second_pass_replaces(self) -> None:
|
||||
command = self.copy(authoritative=True)
|
||||
self.assertEqual(command[:2], ["rsync", "-aP"])
|
||||
self.assertEqual(command[:3], ["rsync", "-aHAX", "-P"])
|
||||
self.assertNotIn("--backup", command)
|
||||
|
||||
def test_it_creates_the_destination(self) -> None:
|
||||
|
||||
@@ -164,6 +164,7 @@ class TestSeedMain(unittest.TestCase):
|
||||
ns.database = "db"
|
||||
ns.username = "user"
|
||||
ns.password = "pass"
|
||||
ns.password_stdin = False
|
||||
parse_args.return_value = ns
|
||||
|
||||
seed_main.main()
|
||||
@@ -195,6 +196,7 @@ class TestSeedMain(unittest.TestCase):
|
||||
ns.database = "db"
|
||||
ns.username = "user"
|
||||
ns.password = "pass"
|
||||
ns.password_stdin = False
|
||||
parse_args.return_value = ns
|
||||
|
||||
seed_main.main()
|
||||
|
||||
Reference in new issue
Block a user