feat(mirror): support SSH MIRRORS, multi-push origin and remote probe
Some checks failed
CI / test-unit (push) Has been cancelled
CI / test-integration (push) Has been cancelled
CI / test-container (push) Has been cancelled
CI / test-e2e (push) Has been cancelled
CI / test-virgin-user (push) Has been cancelled
CI / test-virgin-root (push) Has been cancelled
Some checks failed
CI / test-unit (push) Has been cancelled
CI / test-integration (push) Has been cancelled
CI / test-container (push) Has been cancelled
CI / test-e2e (push) Has been cancelled
CI / test-virgin-user (push) Has been cancelled
CI / test-virgin-root (push) Has been cancelled
- Switch MIRRORS to SSH-based URLs including custom ports/domains
(GitHub, git.veen.world, code.cymais.cloud)
- Extend mirror IO:
- load_config_mirrors filters empty values
- read_mirrors_file now supports:
* "name url" lines
* "url" lines with auto-generated names from URL host (host[:port])
- write_mirrors_file prints full preview content
- Enhance git_remote:
- determine_primary_remote_url used for origin bootstrap
- ensure_origin_remote keeps existing origin URL and
adds all mirror URLs as additional push URLs
- add is_remote_reachable() helper based on `git ls-remote --exit-code`
- Implement non-destructive remote mirror checks in setup_cmd:
- `_probe_mirror()` wraps `git ls-remote` and returns (ok, message)
- `pkgmgr mirror setup --remote` now probes each mirror URL and
prints [OK]/[WARN] with details instead of placeholder text
- Add unit tests for mirror actions:
- test_git_remote: default SSH URL building and primary URL selection
- test_io: config + MIRRORS parsing including auto-named URL-only entries
- test_setup_cmd: probe_mirror success/failure handling
https://chatgpt.com/share/693adee0-aa3c-800f-b72a-98473fdaf760
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import List, Optional
|
||||
from typing import List, Optional, Set
|
||||
|
||||
from pkgmgr.core.command.run import run_command
|
||||
from pkgmgr.core.git import GitError, run_git
|
||||
@@ -87,18 +87,41 @@ def has_origin_remote(repo_dir: str) -> bool:
|
||||
return "origin" in names
|
||||
|
||||
|
||||
def _ensure_push_urls_for_origin(
|
||||
repo_dir: str,
|
||||
mirrors: MirrorMap,
|
||||
preview: bool,
|
||||
) -> None:
|
||||
"""
|
||||
Ensure that all mirror URLs are present as push URLs on 'origin'.
|
||||
"""
|
||||
desired: Set[str] = {url for url in mirrors.values() if url}
|
||||
if not desired:
|
||||
return
|
||||
|
||||
existing_output = _safe_git_output(
|
||||
["remote", "get-url", "--push", "--all", "origin"],
|
||||
cwd=repo_dir,
|
||||
)
|
||||
existing = set(existing_output.splitlines()) if existing_output else set()
|
||||
|
||||
missing = sorted(desired - existing)
|
||||
for url in missing:
|
||||
cmd = f"git remote set-url --add --push origin {url}"
|
||||
if preview:
|
||||
print(f"[PREVIEW] Would run in {repo_dir!r}: {cmd}")
|
||||
else:
|
||||
print(f"[INFO] Adding push URL to 'origin': {url}")
|
||||
run_command(cmd, cwd=repo_dir, preview=False)
|
||||
|
||||
|
||||
def ensure_origin_remote(
|
||||
repo: Repository,
|
||||
ctx: RepoMirrorContext,
|
||||
preview: bool,
|
||||
) -> None:
|
||||
"""
|
||||
Ensure that a usable 'origin' remote exists.
|
||||
|
||||
Priority for choosing URL:
|
||||
1. resolved_mirrors["origin"]
|
||||
2. any resolved mirror (first by name)
|
||||
3. default SSH URL derived from provider/account/repository
|
||||
Ensure that a usable 'origin' remote exists and has all push URLs.
|
||||
"""
|
||||
repo_dir = ctx.repo_dir
|
||||
resolved_mirrors = ctx.resolved_mirrors
|
||||
@@ -109,33 +132,48 @@ def ensure_origin_remote(
|
||||
|
||||
url = determine_primary_remote_url(repo, resolved_mirrors)
|
||||
|
||||
if not url:
|
||||
print(
|
||||
"[WARN] Could not determine URL for 'origin' remote. "
|
||||
"Please configure mirrors or provider/account/repository."
|
||||
)
|
||||
return
|
||||
|
||||
if not has_origin_remote(repo_dir):
|
||||
if not url:
|
||||
print(
|
||||
"[WARN] Could not determine URL for 'origin' remote. "
|
||||
"Please configure mirrors or provider/account/repository."
|
||||
)
|
||||
return
|
||||
|
||||
cmd = f"git remote add origin {url}"
|
||||
if preview:
|
||||
print(f"[PREVIEW] Would run in {repo_dir!r}: {cmd}")
|
||||
else:
|
||||
print(f"[INFO] Adding 'origin' remote in {repo_dir}: {url}")
|
||||
run_command(cmd, cwd=repo_dir, preview=False)
|
||||
return
|
||||
|
||||
current = current_origin_url(repo_dir)
|
||||
if current == url:
|
||||
print(f"[INFO] 'origin' already points to {url} (no change needed).")
|
||||
return
|
||||
|
||||
cmd = f"git remote set-url origin {url}"
|
||||
if preview:
|
||||
print(f"[PREVIEW] Would run in {repo_dir!r}: {cmd}")
|
||||
else:
|
||||
print(
|
||||
f"[INFO] Updating 'origin' remote in {repo_dir} "
|
||||
f"from {current or '<unknown>'} to {url}"
|
||||
)
|
||||
run_command(cmd, cwd=repo_dir, preview=False)
|
||||
current = current_origin_url(repo_dir)
|
||||
if current == url or not url:
|
||||
print(
|
||||
f"[INFO] 'origin' already points to "
|
||||
f"{current or '<unknown>'} (no change needed)."
|
||||
)
|
||||
else:
|
||||
# We do not auto-change origin here, only log the mismatch.
|
||||
print(
|
||||
"[INFO] 'origin' exists with URL "
|
||||
f"{current or '<unknown>'}; not changing to {url}."
|
||||
)
|
||||
|
||||
# Ensure all mirrors are present as push URLs
|
||||
_ensure_push_urls_for_origin(repo_dir, resolved_mirrors, preview)
|
||||
|
||||
|
||||
def is_remote_reachable(url: str, cwd: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Check whether a remote repository is reachable via `git ls-remote`.
|
||||
|
||||
This does NOT modify anything; it only probes the remote.
|
||||
"""
|
||||
workdir = cwd or os.getcwd()
|
||||
try:
|
||||
# --exit-code → non-zero exit code if the remote does not exist
|
||||
run_git(["ls-remote", "--exit-code", url], cwd=workdir)
|
||||
return True
|
||||
except GitError:
|
||||
return False
|
||||
|
||||
@@ -1,61 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from urllib.parse import urlparse
|
||||
from typing import List, Mapping
|
||||
|
||||
from .types import MirrorMap, Repository
|
||||
|
||||
|
||||
def load_config_mirrors(repo: Repository) -> MirrorMap:
|
||||
"""
|
||||
Load mirrors from the repository configuration entry.
|
||||
|
||||
Supported shapes:
|
||||
|
||||
repo["mirrors"] = {
|
||||
"origin": "ssh://git@example.com/...",
|
||||
"backup": "ssh://git@backup/...",
|
||||
}
|
||||
|
||||
or
|
||||
|
||||
repo["mirrors"] = [
|
||||
{"name": "origin", "url": "ssh://git@example.com/..."},
|
||||
{"name": "backup", "url": "ssh://git@backup/..."},
|
||||
]
|
||||
"""
|
||||
mirrors = repo.get("mirrors") or {}
|
||||
result: MirrorMap = {}
|
||||
|
||||
if isinstance(mirrors, dict):
|
||||
for name, url in mirrors.items():
|
||||
if not url:
|
||||
continue
|
||||
result[str(name)] = str(url)
|
||||
if url:
|
||||
result[str(name)] = str(url)
|
||||
return result
|
||||
|
||||
if isinstance(mirrors, list):
|
||||
for entry in mirrors:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
name = entry.get("name")
|
||||
url = entry.get("url")
|
||||
if not name or not url:
|
||||
continue
|
||||
result[str(name)] = str(url)
|
||||
if isinstance(entry, dict):
|
||||
name = entry.get("name")
|
||||
url = entry.get("url")
|
||||
if name and url:
|
||||
result[str(name)] = str(url)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def read_mirrors_file(repo_dir: str, filename: str = "MIRRORS") -> MirrorMap:
|
||||
"""
|
||||
Read mirrors from the MIRRORS file in the repository directory.
|
||||
|
||||
Simple text format:
|
||||
|
||||
# comment
|
||||
origin ssh://git@example.com/account/repo.git
|
||||
backup ssh://git@backup/account/repo.git
|
||||
Supports:
|
||||
NAME URL
|
||||
URL → auto name = hostname
|
||||
"""
|
||||
path = os.path.join(repo_dir, filename)
|
||||
mirrors: MirrorMap = {}
|
||||
@@ -71,10 +48,24 @@ def read_mirrors_file(repo_dir: str, filename: str = "MIRRORS") -> MirrorMap:
|
||||
continue
|
||||
|
||||
parts = stripped.split(None, 1)
|
||||
if len(parts) != 2:
|
||||
# Ignore malformed lines silently
|
||||
|
||||
# Case 1: "name url"
|
||||
if len(parts) == 2:
|
||||
name, url = parts
|
||||
# Case 2: "url" → auto-generate name
|
||||
elif len(parts) == 1:
|
||||
url = parts[0]
|
||||
parsed = urlparse(url)
|
||||
host = (parsed.netloc or "").split(":")[0]
|
||||
base = host or "mirror"
|
||||
name = base
|
||||
i = 2
|
||||
while name in mirrors:
|
||||
name = f"{base}{i}"
|
||||
i += 1
|
||||
else:
|
||||
continue
|
||||
name, url = parts
|
||||
|
||||
mirrors[name] = url
|
||||
except OSError as exc:
|
||||
print(f"[WARN] Could not read MIRRORS file at {path}: {exc}")
|
||||
@@ -88,22 +79,14 @@ def write_mirrors_file(
|
||||
filename: str = "MIRRORS",
|
||||
preview: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Write mirrors to MIRRORS file.
|
||||
|
||||
Existing file is overwritten. In preview mode we only print what would
|
||||
be written.
|
||||
"""
|
||||
path = os.path.join(repo_dir, filename)
|
||||
lines: List[str] = [f"{name} {url}" for name, url in sorted(mirrors.items())]
|
||||
lines = [f"{name} {url}" for name, url in sorted(mirrors.items())]
|
||||
content = "\n".join(lines) + ("\n" if lines else "")
|
||||
|
||||
if preview:
|
||||
print(f"[PREVIEW] Would write MIRRORS file at {path}:")
|
||||
if content:
|
||||
print(content.rstrip())
|
||||
else:
|
||||
print("(empty)")
|
||||
print(content or "(empty)")
|
||||
return
|
||||
|
||||
try:
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
from typing import List, Tuple
|
||||
|
||||
from pkgmgr.core.git import run_git, GitError
|
||||
|
||||
from .context import build_context
|
||||
from .git_remote import determine_primary_remote_url, ensure_origin_remote
|
||||
@@ -13,6 +15,9 @@ def _setup_local_mirrors_for_repo(
|
||||
all_repos: List[Repository],
|
||||
preview: bool,
|
||||
) -> None:
|
||||
"""
|
||||
Ensure local Git state is sane (currently: 'origin' remote).
|
||||
"""
|
||||
ctx = build_context(repo, repositories_base_dir, all_repos)
|
||||
|
||||
print("------------------------------------------------------------")
|
||||
@@ -24,6 +29,27 @@ def _setup_local_mirrors_for_repo(
|
||||
print()
|
||||
|
||||
|
||||
def _probe_mirror(url: str, repo_dir: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Probe a remote mirror by running `git ls-remote <url>`.
|
||||
|
||||
Returns:
|
||||
(True, "") on success,
|
||||
(False, error_message) on failure.
|
||||
|
||||
Wichtig:
|
||||
- Wir werten ausschließlich den Exit-Code aus.
|
||||
- STDERR kann Hinweise/Warnings enthalten und ist NICHT automatisch ein Fehler.
|
||||
"""
|
||||
try:
|
||||
# Wir ignorieren stdout komplett; wichtig ist nur, dass der Befehl ohne
|
||||
# GitError (also Exit-Code 0) durchläuft.
|
||||
run_git(["ls-remote", url], cwd=repo_dir)
|
||||
return True, ""
|
||||
except GitError as exc:
|
||||
return False, str(exc)
|
||||
|
||||
|
||||
def _setup_remote_mirrors_for_repo(
|
||||
repo: Repository,
|
||||
repositories_base_dir: str,
|
||||
@@ -31,45 +57,75 @@ def _setup_remote_mirrors_for_repo(
|
||||
preview: bool,
|
||||
) -> None:
|
||||
"""
|
||||
Placeholder for remote-side setup.
|
||||
Remote-side setup / validation.
|
||||
|
||||
This is intentionally conservative:
|
||||
- We *do not* call any provider APIs automatically here.
|
||||
- Instead, we show what should exist and which URL should be created.
|
||||
Aktuell werden nur **nicht-destruktive Checks** gemacht:
|
||||
|
||||
- Für jeden Mirror (aus config + MIRRORS-Datei, file gewinnt):
|
||||
* `git ls-remote <url>` wird ausgeführt.
|
||||
* Bei Exit-Code 0 → [OK]
|
||||
* Bei Fehler → [WARN] + Details aus der GitError-Exception
|
||||
|
||||
Es werden **keine** Provider-APIs aufgerufen und keine Repos angelegt.
|
||||
"""
|
||||
ctx = build_context(repo, repositories_base_dir, all_repos)
|
||||
resolved_m = ctx.resolved_mirrors
|
||||
|
||||
primary_url = determine_primary_remote_url(repo, resolved_m)
|
||||
|
||||
print("------------------------------------------------------------")
|
||||
print(f"[MIRROR SETUP:REMOTE] {ctx.identifier}")
|
||||
print(f"[MIRROR SETUP:REMOTE] dir: {ctx.repo_dir}")
|
||||
print("------------------------------------------------------------")
|
||||
|
||||
if not primary_url:
|
||||
if not resolved_m:
|
||||
# Optional: Fallback auf eine heuristisch bestimmte URL, falls wir
|
||||
# irgendwann "automatisch anlegen" implementieren wollen.
|
||||
primary_url = determine_primary_remote_url(repo, resolved_m)
|
||||
if not primary_url:
|
||||
print(
|
||||
"[INFO] No mirrors configured (config or MIRRORS file), and no "
|
||||
"primary URL could be derived from provider/account/repository."
|
||||
)
|
||||
print()
|
||||
return
|
||||
|
||||
ok, error_message = _probe_mirror(primary_url, ctx.repo_dir)
|
||||
if ok:
|
||||
print(f"[OK] Remote mirror (primary) is reachable: {primary_url}")
|
||||
else:
|
||||
print("[WARN] Primary remote URL is NOT reachable:")
|
||||
print(f" {primary_url}")
|
||||
if error_message:
|
||||
print(" Details:")
|
||||
for line in error_message.splitlines():
|
||||
print(f" {line}")
|
||||
|
||||
print()
|
||||
print(
|
||||
"[WARN] Could not determine primary remote URL for this repository.\n"
|
||||
" Please ensure provider/account/repository and/or mirrors "
|
||||
"are set in your config."
|
||||
"[INFO] Remote checks are non-destructive and only use `git ls-remote` "
|
||||
"to probe mirror URLs."
|
||||
)
|
||||
print()
|
||||
return
|
||||
|
||||
if preview:
|
||||
print(
|
||||
"[PREVIEW] Would ensure that a remote repository exists for:\n"
|
||||
f" {primary_url}\n"
|
||||
" (Provider-specific API calls not implemented yet.)"
|
||||
)
|
||||
else:
|
||||
print(
|
||||
"[INFO] Remote-setup logic is not implemented yet.\n"
|
||||
" Please create the remote repository manually if needed:\n"
|
||||
f" {primary_url}\n"
|
||||
)
|
||||
# Normaler Fall: wir haben benannte Mirrors aus config/MIRRORS
|
||||
for name, url in sorted(resolved_m.items()):
|
||||
ok, error_message = _probe_mirror(url, ctx.repo_dir)
|
||||
if ok:
|
||||
print(f"[OK] Remote mirror '{name}' is reachable: {url}")
|
||||
else:
|
||||
print(f"[WARN] Remote mirror '{name}' is NOT reachable:")
|
||||
print(f" {url}")
|
||||
if error_message:
|
||||
print(" Details:")
|
||||
for line in error_message.splitlines():
|
||||
print(f" {line}")
|
||||
|
||||
print()
|
||||
print(
|
||||
"[INFO] Remote checks are non-destructive and only use `git ls-remote` "
|
||||
"to probe mirror URLs."
|
||||
)
|
||||
print()
|
||||
|
||||
|
||||
def setup_mirrors(
|
||||
@@ -88,8 +144,8 @@ def setup_mirrors(
|
||||
points to a reasonable URL).
|
||||
|
||||
remote:
|
||||
- Placeholder that prints what should exist on the remote side.
|
||||
Actual API calls to providers are not implemented yet.
|
||||
- Non-destructive remote checks using `git ls-remote` for each mirror URL.
|
||||
Es werden keine Repositories auf dem Provider angelegt.
|
||||
"""
|
||||
for repo in selected_repos:
|
||||
if local:
|
||||
|
||||
Reference in New Issue
Block a user