refactor: remove bootstrap wrappers and split into config/service/installers
- Replace bootstrap wrapper with config-driven service orchestration - Introduce Config dataclass for centralized env/CLI validation - Add MatomoApi service for session login + app token creation - Move Playwright installer into installers/web and drop old install package - Refactor HttpClient to unify HTTP handling and debug to stderr - Make E2E tests use sys.executable instead of hardcoded python3
This commit is contained in:
Kevin Veen-Birkenbach
@@ -1,16 +1,25 @@
|
||||
from .cli import parse_args
|
||||
from .bootstrap import run_bootstrap
|
||||
from .errors import BootstrapError
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from .cli import parse_args
|
||||
from .config import config_from_env_and_args
|
||||
from .errors import BootstrapError
|
||||
from .service import run
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
try:
|
||||
token = run_bootstrap(args)
|
||||
config = config_from_env_and_args(args)
|
||||
token = run(config)
|
||||
print(token)
|
||||
return 0
|
||||
except ValueError as exc:
|
||||
# config validation errors
|
||||
print(f"[ERROR] {exc}", file=sys.stderr)
|
||||
return 2
|
||||
except BootstrapError as exc:
|
||||
print(f"[ERROR] {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
|
||||
from .errors import TokenCreationError
|
||||
from .http import HttpClient
|
||||
|
||||
|
||||
def _md5(text: str) -> str:
|
||||
return hashlib.md5(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _try_json(body: str) -> object:
|
||||
try:
|
||||
return json.loads(body)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise TokenCreationError(f"Invalid JSON from Matomo API: {body[:400]}") from exc
|
||||
|
||||
|
||||
def _dbg(msg: str, enabled: bool) -> None:
|
||||
if enabled:
|
||||
# IMPORTANT: keep stdout clean (tests expect only token on stdout)
|
||||
print(msg, file=sys.stderr)
|
||||
|
||||
|
||||
def _login_via_logme(client: HttpClient, admin_user: str, admin_password: str, debug: bool) -> None:
|
||||
"""
|
||||
Create an authenticated Matomo session (cookie jar) using the classic Login controller.
|
||||
|
||||
Matomo accepts the md5 hashed password in the `password` parameter for action=logme.
|
||||
We rely on urllib's opener to follow redirects and store cookies.
|
||||
"""
|
||||
md5_password = _md5(admin_password)
|
||||
|
||||
try:
|
||||
status, body = client.get(
|
||||
"/index.php",
|
||||
{
|
||||
"module": "Login",
|
||||
"action": "logme",
|
||||
"login": admin_user,
|
||||
"password": md5_password,
|
||||
},
|
||||
)
|
||||
_dbg(f"[auth] login via logme returned HTTP {status} (body preview: {body[:120]!r})", debug)
|
||||
except urllib.error.HTTPError as exc:
|
||||
# Even 4xx/5xx can still set cookies; continue and let the API call validate.
|
||||
try:
|
||||
err_body = exc.read().decode("utf-8", errors="replace")
|
||||
except Exception:
|
||||
err_body = ""
|
||||
_dbg(f"[auth] login via logme raised HTTPError {exc.code} (body preview: {err_body[:120]!r})", debug)
|
||||
|
||||
|
||||
def create_app_token_via_session(
|
||||
*,
|
||||
client: HttpClient,
|
||||
admin_user: str,
|
||||
admin_password: str,
|
||||
description: str,
|
||||
debug: bool = False,
|
||||
) -> str:
|
||||
"""
|
||||
Create an app-specific token using an authenticated SESSION (cookies),
|
||||
not via UsersManager.getTokenAuth (removed/not available in Matomo 5.3.x images).
|
||||
|
||||
If MATOMO_BOOTSTRAP_TOKEN_AUTH is already set, we return it.
|
||||
"""
|
||||
env_token = os.environ.get("MATOMO_BOOTSTRAP_TOKEN_AUTH")
|
||||
if env_token:
|
||||
_dbg("[auth] Using MATOMO_BOOTSTRAP_TOKEN_AUTH from environment.", debug)
|
||||
return env_token
|
||||
|
||||
# 1) Establish logged-in session
|
||||
_login_via_logme(client, admin_user, admin_password, debug=debug)
|
||||
|
||||
# 2) Use the session cookie to create an app specific token
|
||||
status, body = client.post(
|
||||
"/index.php",
|
||||
{
|
||||
"module": "API",
|
||||
"method": "UsersManager.createAppSpecificTokenAuth",
|
||||
"userLogin": admin_user,
|
||||
"passwordConfirmation": admin_password,
|
||||
"description": description,
|
||||
"format": "json",
|
||||
},
|
||||
)
|
||||
|
||||
_dbg(f"[auth] createAppSpecificTokenAuth HTTP {status} body[:200]={body[:200]!r}", debug)
|
||||
|
||||
if status != 200:
|
||||
raise TokenCreationError(f"HTTP {status} during token creation: {body[:400]}")
|
||||
|
||||
data = _try_json(body)
|
||||
|
||||
token = data.get("value") if isinstance(data, dict) else None
|
||||
if not token:
|
||||
raise TokenCreationError(f"Unexpected response from token creation: {data}")
|
||||
|
||||
return str(token)
|
||||
@@ -1,37 +0,0 @@
|
||||
from argparse import Namespace
|
||||
|
||||
from .api_tokens import create_app_token_via_session
|
||||
from .health import assert_matomo_ready
|
||||
from .http import HttpClient
|
||||
from .install.web_installer import ensure_installed
|
||||
|
||||
|
||||
def run_bootstrap(args: Namespace) -> str:
|
||||
# 1) Ensure Matomo is installed (NO-OP if already installed)
|
||||
ensure_installed(
|
||||
base_url=args.base_url,
|
||||
admin_user=args.admin_user,
|
||||
admin_password=args.admin_password,
|
||||
admin_email=args.admin_email,
|
||||
debug=args.debug,
|
||||
)
|
||||
|
||||
# 2) Now the UI/API should be reachable and "installed"
|
||||
assert_matomo_ready(args.base_url, timeout=args.timeout)
|
||||
|
||||
# 3) Create app-specific token via authenticated session (cookie-based)
|
||||
client = HttpClient(
|
||||
base_url=args.base_url,
|
||||
timeout=args.timeout,
|
||||
debug=args.debug,
|
||||
)
|
||||
|
||||
token = create_app_token_via_session(
|
||||
client=client,
|
||||
admin_user=args.admin_user,
|
||||
admin_password=args.admin_password,
|
||||
description=args.token_description,
|
||||
debug=args.debug,
|
||||
)
|
||||
|
||||
return token
|
||||
@@ -30,23 +30,21 @@ def parse_args() -> argparse.Namespace:
|
||||
p.add_argument(
|
||||
"--token-description",
|
||||
default=os.environ.get("MATOMO_TOKEN_DESCRIPTION", "matomo-bootstrap"),
|
||||
help="App token description",
|
||||
)
|
||||
p.add_argument("--timeout", type=int, default=int(os.environ.get("MATOMO_TIMEOUT", "20")))
|
||||
p.add_argument("--debug", action="store_true")
|
||||
p.add_argument(
|
||||
"--timeout",
|
||||
type=int,
|
||||
default=int(os.environ.get("MATOMO_TIMEOUT", "20")),
|
||||
help="Network timeout in seconds (or MATOMO_TIMEOUT env)",
|
||||
)
|
||||
p.add_argument("--debug", action="store_true", help="Enable debug logs on stderr")
|
||||
|
||||
args = p.parse_args()
|
||||
# Optional (future use)
|
||||
p.add_argument(
|
||||
"--matomo-container-name",
|
||||
default=os.environ.get("MATOMO_CONTAINER_NAME"),
|
||||
help="Matomo container name (optional; also MATOMO_CONTAINER_NAME env)",
|
||||
)
|
||||
|
||||
missing = []
|
||||
if not args.base_url:
|
||||
missing.append("--base-url (or MATOMO_URL)")
|
||||
if not args.admin_user:
|
||||
missing.append("--admin-user (or MATOMO_ADMIN_USER)")
|
||||
if not args.admin_password:
|
||||
missing.append("--admin-password (or MATOMO_ADMIN_PASSWORD)")
|
||||
if not args.admin_email:
|
||||
missing.append("--admin-email (or MATOMO_ADMIN_EMAIL)")
|
||||
|
||||
if missing:
|
||||
p.error("missing required values: " + ", ".join(missing))
|
||||
|
||||
return args
|
||||
return p.parse_args()
|
||||
|
||||
65
src/matomo_bootstrap/config.py
Normal file
65
src/matomo_bootstrap/config.py
Normal file
@@ -0,0 +1,65 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import os
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
base_url: str
|
||||
admin_user: str
|
||||
admin_password: str
|
||||
admin_email: str
|
||||
token_description: str = "matomo-bootstrap"
|
||||
timeout: int = 20
|
||||
debug: bool = False
|
||||
matomo_container_name: str | None = None # optional, for future console installer usage
|
||||
|
||||
|
||||
def config_from_env_and_args(args) -> Config:
|
||||
"""
|
||||
Build a Config object from CLI args (preferred) and environment variables (fallback).
|
||||
"""
|
||||
base_url = getattr(args, "base_url", None) or os.environ.get("MATOMO_URL")
|
||||
admin_user = getattr(args, "admin_user", None) or os.environ.get("MATOMO_ADMIN_USER")
|
||||
admin_password = getattr(args, "admin_password", None) or os.environ.get("MATOMO_ADMIN_PASSWORD")
|
||||
admin_email = getattr(args, "admin_email", None) or os.environ.get("MATOMO_ADMIN_EMAIL")
|
||||
|
||||
token_description = (
|
||||
getattr(args, "token_description", None)
|
||||
or os.environ.get("MATOMO_TOKEN_DESCRIPTION")
|
||||
or "matomo-bootstrap"
|
||||
)
|
||||
|
||||
timeout = int(getattr(args, "timeout", None) or os.environ.get("MATOMO_TIMEOUT") or "20")
|
||||
debug = bool(getattr(args, "debug", False))
|
||||
|
||||
matomo_container_name = (
|
||||
getattr(args, "matomo_container_name", None)
|
||||
or os.environ.get("MATOMO_CONTAINER_NAME")
|
||||
or None
|
||||
)
|
||||
|
||||
missing: list[str] = []
|
||||
if not base_url:
|
||||
missing.append("--base-url (or MATOMO_URL)")
|
||||
if not admin_user:
|
||||
missing.append("--admin-user (or MATOMO_ADMIN_USER)")
|
||||
if not admin_password:
|
||||
missing.append("--admin-password (or MATOMO_ADMIN_PASSWORD)")
|
||||
if not admin_email:
|
||||
missing.append("--admin-email (or MATOMO_ADMIN_EMAIL)")
|
||||
|
||||
if missing:
|
||||
raise ValueError("missing required values: " + ", ".join(missing))
|
||||
|
||||
return Config(
|
||||
base_url=str(base_url),
|
||||
admin_user=str(admin_user),
|
||||
admin_password=str(admin_password),
|
||||
admin_email=str(admin_email),
|
||||
token_description=str(token_description),
|
||||
timeout=timeout,
|
||||
debug=debug,
|
||||
matomo_container_name=matomo_container_name,
|
||||
)
|
||||
@@ -1,4 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import urllib.request
|
||||
|
||||
from .errors import MatomoNotReadyError
|
||||
|
||||
|
||||
@@ -9,5 +12,6 @@ def assert_matomo_ready(base_url: str, timeout: int = 10) -> None:
|
||||
except Exception as exc:
|
||||
raise MatomoNotReadyError(f"Matomo not reachable: {exc}") from exc
|
||||
|
||||
if "Matomo" not in html and "piwik" not in html.lower():
|
||||
lower = html.lower()
|
||||
if "matomo" not in lower and "piwik" not in lower:
|
||||
raise MatomoNotReadyError("Matomo UI not detected at base URL")
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import http.cookiejar
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
@@ -16,15 +19,11 @@ class HttpClient:
|
||||
urllib.request.HTTPCookieProcessor(self.cookies)
|
||||
)
|
||||
|
||||
def get(self, path: str, params: Dict[str, str]) -> Tuple[int, str]:
|
||||
qs = urllib.parse.urlencode(params)
|
||||
url = f"{self.base_url}{path}?{qs}"
|
||||
|
||||
def _dbg(self, msg: str) -> None:
|
||||
if self.debug:
|
||||
print(f"[HTTP] GET {url}")
|
||||
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
print(msg, file=sys.stderr)
|
||||
|
||||
def _open(self, req: urllib.request.Request) -> Tuple[int, str]:
|
||||
try:
|
||||
with self.opener.open(req, timeout=self.timeout) as resp:
|
||||
body = resp.read().decode("utf-8", errors="replace")
|
||||
@@ -37,22 +36,25 @@ class HttpClient:
|
||||
body = str(exc)
|
||||
return exc.code, body
|
||||
|
||||
def get(self, path: str, params: Dict[str, str]) -> Tuple[int, str]:
|
||||
qs = urllib.parse.urlencode(params)
|
||||
if path == "/":
|
||||
url = f"{self.base_url}/"
|
||||
else:
|
||||
url = f"{self.base_url}{path}"
|
||||
if qs:
|
||||
url = f"{url}?{qs}"
|
||||
|
||||
self._dbg(f"[HTTP] GET {url}")
|
||||
|
||||
req = urllib.request.Request(url, method="GET")
|
||||
return self._open(req)
|
||||
|
||||
def post(self, path: str, data: Dict[str, str]) -> Tuple[int, str]:
|
||||
url = self.base_url + path
|
||||
encoded = urllib.parse.urlencode(data).encode()
|
||||
|
||||
if self.debug:
|
||||
print(f"[HTTP] POST {url} keys={list(data.keys())}")
|
||||
self._dbg(f"[HTTP] POST {url} keys={list(data.keys())}")
|
||||
|
||||
req = urllib.request.Request(url, data=encoded, method="POST")
|
||||
|
||||
try:
|
||||
with self.opener.open(req, timeout=self.timeout) as resp:
|
||||
body = resp.read().decode("utf-8", errors="replace")
|
||||
return resp.status, body
|
||||
except urllib.error.HTTPError as exc:
|
||||
try:
|
||||
body = exc.read().decode("utf-8", errors="replace")
|
||||
except Exception:
|
||||
body = str(exc)
|
||||
return exc.code, body
|
||||
return self._open(req)
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
|
||||
MATOMO_ROOT = "/var/www/html"
|
||||
CONSOLE = f"{MATOMO_ROOT}/console"
|
||||
|
||||
|
||||
def _run(cmd: list[str]) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(cmd, text=True, capture_output=True)
|
||||
|
||||
|
||||
def _container_state(container_name: str) -> str:
|
||||
res = _run(["docker", "inspect", "-f", "{{.State.Status}}", container_name])
|
||||
return (res.stdout or "").strip()
|
||||
|
||||
|
||||
def _wait_container_running(container_name: str, timeout: int = 60) -> None:
|
||||
last = ""
|
||||
for _ in range(timeout):
|
||||
state = _container_state(container_name)
|
||||
last = state
|
||||
if state == "running":
|
||||
return
|
||||
time.sleep(1)
|
||||
raise RuntimeError(f"Container '{container_name}' did not become running (last state: {last})")
|
||||
|
||||
|
||||
def _exec(container_name: str, argv: list[str]) -> subprocess.CompletedProcess:
|
||||
return _run(["docker", "exec", container_name, *argv])
|
||||
|
||||
|
||||
def _sh(container_name: str, script: str) -> subprocess.CompletedProcess:
|
||||
# Use sh -lc so PATH + cwd behave more like interactive container sessions
|
||||
return _exec(container_name, ["sh", "-lc", script])
|
||||
|
||||
|
||||
def _console_exists(container_name: str) -> bool:
|
||||
res = _sh(container_name, f"test -x {CONSOLE} && echo yes || echo no")
|
||||
return (res.stdout or "").strip() == "yes"
|
||||
|
||||
|
||||
def _is_installed(container_name: str) -> bool:
|
||||
res = _sh(container_name, f"test -f {MATOMO_ROOT}/config/config.ini.php && echo yes || echo no")
|
||||
return (res.stdout or "").strip() == "yes"
|
||||
|
||||
|
||||
def _console_list(container_name: str) -> str:
|
||||
# --no-ansi for stable parsing
|
||||
res = _sh(container_name, f"{CONSOLE} list --no-ansi 2>/dev/null || true")
|
||||
return (res.stdout or "") + "\n" + (res.stderr or "")
|
||||
|
||||
|
||||
def _has_command(console_list_output: str, command: str) -> bool:
|
||||
# cheap but robust enough
|
||||
return f" {command} " in console_list_output or f"\n{command}\n" in console_list_output or command in console_list_output
|
||||
|
||||
|
||||
def ensure_installed_via_console(
|
||||
*,
|
||||
container_name: str,
|
||||
admin_user: str,
|
||||
admin_password: str,
|
||||
admin_email: str,
|
||||
debug: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Ensure Matomo is installed using the container's console if possible.
|
||||
If no known install command exists, we do NOT guess: we raise with diagnostics.
|
||||
"""
|
||||
_wait_container_running(container_name, timeout=90)
|
||||
|
||||
if _is_installed(container_name):
|
||||
if debug:
|
||||
print("[install] Matomo already installed (config.ini.php exists).")
|
||||
return
|
||||
|
||||
if not _console_exists(container_name):
|
||||
raise RuntimeError(f"Matomo console not found/executable at {CONSOLE} inside container '{container_name}'.")
|
||||
|
||||
listing = _console_list(container_name)
|
||||
if debug:
|
||||
print("[install] Matomo console list obtained.")
|
||||
|
||||
# Matomo versions differ; we discover what exists.
|
||||
# Historically: core:install. Your earlier log showed it does NOT exist in 5.3.2 image.
|
||||
# Therefore we refuse to guess and provide the list in the exception.
|
||||
if _has_command(listing, "core:install"):
|
||||
# If this ever exists, use it.
|
||||
cmd = (
|
||||
f"{CONSOLE} core:install --no-ansi "
|
||||
f"--database-host=db "
|
||||
f"--database-username=matomo "
|
||||
f"--database-password=matomo_pw "
|
||||
f"--database-name=matomo "
|
||||
f"--login={admin_user} "
|
||||
f"--password={admin_password} "
|
||||
f"--email={admin_email} "
|
||||
f"--url=http://localhost "
|
||||
)
|
||||
res = _sh(container_name, cmd)
|
||||
if res.returncode != 0:
|
||||
raise RuntimeError(f"Matomo CLI install failed.\nexit={res.returncode}\nstdout:\n{res.stdout}\nstderr:\n{res.stderr}")
|
||||
return
|
||||
|
||||
# No install command -> fail with diagnostics (don’t keep burning time).
|
||||
raise RuntimeError(
|
||||
"Matomo is not installed yet, but no supported CLI install command was found in this image.\n"
|
||||
"This Matomo image likely expects the web installer.\n"
|
||||
"\n[console list]\n"
|
||||
f"{listing}\n"
|
||||
)
|
||||
@@ -1,227 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
# Optional knobs (mostly for debugging / CI stability)
|
||||
PLAYWRIGHT_HEADLESS = (
|
||||
os.environ.get("MATOMO_PLAYWRIGHT_HEADLESS", "1").strip() not in ("0", "false", "False")
|
||||
)
|
||||
PLAYWRIGHT_SLOWMO_MS = int(os.environ.get("MATOMO_PLAYWRIGHT_SLOWMO_MS", "0"))
|
||||
PLAYWRIGHT_NAV_TIMEOUT_MS = int(os.environ.get("MATOMO_PLAYWRIGHT_NAV_TIMEOUT_MS", "60000"))
|
||||
|
||||
# Values used by the installer flow (recorded)
|
||||
DEFAULT_SITE_NAME = os.environ.get("MATOMO_SITE_NAME", "localhost")
|
||||
DEFAULT_SITE_URL = os.environ.get("MATOMO_SITE_URL", "http://localhost")
|
||||
DEFAULT_TIMEZONE = os.environ.get("MATOMO_TIMEZONE", "Germany - Berlin")
|
||||
DEFAULT_ECOMMERCE = os.environ.get("MATOMO_ECOMMERCE", "Ecommerce enabled")
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
# IMPORTANT: logs must not pollute stdout (tests expect only token on stdout)
|
||||
print(msg, file=sys.stderr)
|
||||
|
||||
|
||||
def wait_http(url: str, timeout: int = 180) -> None:
|
||||
"""
|
||||
Consider Matomo 'reachable' as soon as the HTTP server answers - even with 500.
|
||||
urllib raises HTTPError for 4xx/5xx, so we must treat that as reachability too.
|
||||
"""
|
||||
_log(f"[install] Waiting for Matomo HTTP at {url} ...")
|
||||
last_err: Exception | None = None
|
||||
|
||||
for i in range(timeout):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as resp:
|
||||
_ = resp.read(128)
|
||||
_log("[install] Matomo HTTP reachable (2xx/3xx).")
|
||||
return
|
||||
except urllib.error.HTTPError as exc:
|
||||
# 4xx/5xx means the server answered -> reachable
|
||||
_log(f"[install] Matomo HTTP reachable (HTTP {exc.code}).")
|
||||
return
|
||||
except Exception as exc:
|
||||
last_err = exc
|
||||
if i % 5 == 0:
|
||||
_log(f"[install] still waiting ({i}/{timeout}) … ({type(exc).__name__})")
|
||||
time.sleep(1)
|
||||
|
||||
raise RuntimeError(f"Matomo did not become reachable after {timeout}s: {url} ({last_err})")
|
||||
|
||||
|
||||
def is_installed(url: str) -> bool:
|
||||
"""
|
||||
Heuristic:
|
||||
- installed instances typically render login module links
|
||||
- installer renders 'installation' wizard content
|
||||
"""
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=5) as resp:
|
||||
html = resp.read().decode(errors="ignore").lower()
|
||||
return ("module=login" in html) or ("matomo › login" in html) or ("matomo/login" in html)
|
||||
except urllib.error.HTTPError as exc:
|
||||
try:
|
||||
html = exc.read().decode(errors="ignore").lower()
|
||||
return ("module=login" in html) or ("matomo › login" in html) or ("matomo/login" in html)
|
||||
except Exception:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def ensure_installed(
|
||||
base_url: str,
|
||||
admin_user: str,
|
||||
admin_password: str,
|
||||
admin_email: str,
|
||||
debug: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Ensure Matomo is installed.
|
||||
NO-OP if already installed.
|
||||
|
||||
This implementation ONLY uses the Playwright web installer (recorded flow).
|
||||
"""
|
||||
wait_http(base_url)
|
||||
|
||||
if is_installed(base_url):
|
||||
if debug:
|
||||
_log("[install] Matomo already looks installed. Skipping installer.")
|
||||
return
|
||||
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
_log("[install] Running Matomo web installer via Playwright (recorded flow)...")
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(
|
||||
headless=PLAYWRIGHT_HEADLESS,
|
||||
slow_mo=PLAYWRIGHT_SLOWMO_MS if PLAYWRIGHT_SLOWMO_MS > 0 else None,
|
||||
)
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
page.set_default_navigation_timeout(PLAYWRIGHT_NAV_TIMEOUT_MS)
|
||||
page.set_default_timeout(PLAYWRIGHT_NAV_TIMEOUT_MS)
|
||||
|
||||
def _dbg(msg: str) -> None:
|
||||
if debug:
|
||||
_log(f"[install] {msg}")
|
||||
|
||||
def click_next() -> None:
|
||||
"""
|
||||
Matomo installer mixes link/button variants and sometimes includes '»'.
|
||||
We try common variants in a robust order.
|
||||
"""
|
||||
candidates = [
|
||||
("link", "Next »"),
|
||||
("button", "Next »"),
|
||||
("link", "Next"),
|
||||
("button", "Next"),
|
||||
("link", "Continue"),
|
||||
("button", "Continue"),
|
||||
("link", "Proceed"),
|
||||
("button", "Proceed"),
|
||||
("link", "Start Installation"),
|
||||
("button", "Start Installation"),
|
||||
("link", "Weiter"),
|
||||
("button", "Weiter"),
|
||||
("link", "Fortfahren"),
|
||||
("button", "Fortfahren"),
|
||||
]
|
||||
|
||||
for role, name in candidates:
|
||||
loc = page.get_by_role(role, name=name)
|
||||
if loc.count() > 0:
|
||||
_dbg(f"click_next(): {role} '{name}'")
|
||||
loc.first.click()
|
||||
return
|
||||
|
||||
# last resort: some pages use same text but different element types
|
||||
loc = page.get_by_text("Next", exact=False)
|
||||
if loc.count() > 0:
|
||||
_dbg("click_next(): fallback text 'Next'")
|
||||
loc.first.click()
|
||||
return
|
||||
|
||||
raise RuntimeError("Could not find a Next/Continue control in the installer UI.")
|
||||
|
||||
# --- Recorded-ish flow, but made variable-based + more stable ---
|
||||
page.goto(base_url, wait_until="domcontentloaded")
|
||||
|
||||
def superuser_form_visible() -> bool:
|
||||
# In your recording, the superuser "Login" field was "#login-0".
|
||||
return page.locator("#login-0").count() > 0
|
||||
|
||||
# Click next until the superuser page shows up (cap to avoid infinite loops).
|
||||
for _ in range(12):
|
||||
if superuser_form_visible():
|
||||
break
|
||||
click_next()
|
||||
page.wait_for_load_state("domcontentloaded")
|
||||
page.wait_for_timeout(200)
|
||||
else:
|
||||
raise RuntimeError("Installer did not reach superuser step (login-0 not found).")
|
||||
|
||||
# Superuser step
|
||||
page.locator("#login-0").click()
|
||||
page.locator("#login-0").fill(admin_user)
|
||||
|
||||
page.locator("#password-0").click()
|
||||
page.locator("#password-0").fill(admin_password)
|
||||
|
||||
# Repeat password (some versions have it)
|
||||
if page.locator("#password_bis-0").count() > 0:
|
||||
page.locator("#password_bis-0").click()
|
||||
page.locator("#password_bis-0").fill(admin_password)
|
||||
|
||||
page.locator("#email-0").click()
|
||||
page.locator("#email-0").fill(admin_email)
|
||||
|
||||
# Next
|
||||
if page.get_by_role("button", name="Next »").count() > 0:
|
||||
page.get_by_role("button", name="Next »").click()
|
||||
else:
|
||||
click_next()
|
||||
|
||||
# First website
|
||||
if page.locator("#siteName-0").count() > 0:
|
||||
page.locator("#siteName-0").click()
|
||||
page.locator("#siteName-0").fill(DEFAULT_SITE_NAME)
|
||||
|
||||
if page.locator("#url-0").count() > 0:
|
||||
page.locator("#url-0").click()
|
||||
page.locator("#url-0").fill(DEFAULT_SITE_URL)
|
||||
|
||||
# Timezone dropdown (best-effort)
|
||||
try:
|
||||
page.get_by_role("combobox").first.click()
|
||||
page.get_by_role("listbox").get_by_text(DEFAULT_TIMEZONE).click()
|
||||
except Exception:
|
||||
_dbg("Timezone selection skipped (not found / changed UI).")
|
||||
|
||||
# Ecommerce dropdown (best-effort)
|
||||
try:
|
||||
page.get_by_role("combobox").nth(2).click()
|
||||
page.get_by_role("listbox").get_by_text(DEFAULT_ECOMMERCE).click()
|
||||
except Exception:
|
||||
_dbg("Ecommerce selection skipped (not found / changed UI).")
|
||||
|
||||
# Next pages to finish
|
||||
click_next()
|
||||
page.wait_for_load_state("domcontentloaded")
|
||||
|
||||
if page.get_by_role("link", name="Next »").count() > 0:
|
||||
page.get_by_role("link", name="Next »").click()
|
||||
|
||||
if page.get_by_role("button", name="Continue to Matomo »").count() > 0:
|
||||
page.get_by_role("button", name="Continue to Matomo »").click()
|
||||
|
||||
context.close()
|
||||
browser.close()
|
||||
|
||||
time.sleep(1)
|
||||
if not is_installed(base_url):
|
||||
raise RuntimeError("[install] Installer did not reach installed state.")
|
||||
|
||||
_log("[install] Installation finished.")
|
||||
1
src/matomo_bootstrap/installers/__init__.py
Normal file
1
src/matomo_bootstrap/installers/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
__all__ = []
|
||||
11
src/matomo_bootstrap/installers/base.py
Normal file
11
src/matomo_bootstrap/installers/base.py
Normal file
@@ -0,0 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from ..config import Config
|
||||
|
||||
|
||||
class Installer(ABC):
|
||||
@abstractmethod
|
||||
def ensure_installed(self, config: Config) -> None:
|
||||
raise NotImplementedError
|
||||
216
src/matomo_bootstrap/installers/web.py
Normal file
216
src/matomo_bootstrap/installers/web.py
Normal file
@@ -0,0 +1,216 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from .base import Installer
|
||||
from ..config import Config
|
||||
|
||||
|
||||
# Optional knobs (mostly for debugging / CI stability)
|
||||
PLAYWRIGHT_HEADLESS = (
|
||||
os.environ.get("MATOMO_PLAYWRIGHT_HEADLESS", "1").strip() not in ("0", "false", "False")
|
||||
)
|
||||
PLAYWRIGHT_SLOWMO_MS = int(os.environ.get("MATOMO_PLAYWRIGHT_SLOWMO_MS", "0"))
|
||||
PLAYWRIGHT_NAV_TIMEOUT_MS = int(os.environ.get("MATOMO_PLAYWRIGHT_NAV_TIMEOUT_MS", "60000"))
|
||||
|
||||
# Values used by the installer flow (recorded)
|
||||
DEFAULT_SITE_NAME = os.environ.get("MATOMO_SITE_NAME", "localhost")
|
||||
DEFAULT_SITE_URL = os.environ.get("MATOMO_SITE_URL", "http://localhost")
|
||||
DEFAULT_TIMEZONE = os.environ.get("MATOMO_TIMEZONE", "Germany - Berlin")
|
||||
DEFAULT_ECOMMERCE = os.environ.get("MATOMO_ECOMMERCE", "Ecommerce enabled")
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
# IMPORTANT: logs must not pollute stdout (tests expect only token on stdout)
|
||||
print(msg, file=sys.stderr)
|
||||
|
||||
|
||||
def wait_http(url: str, timeout: int = 180) -> None:
|
||||
"""
|
||||
Consider Matomo 'reachable' as soon as the HTTP server answers - even with 500.
|
||||
urllib raises HTTPError for 4xx/5xx, so we must treat that as reachability too.
|
||||
"""
|
||||
_log(f"[install] Waiting for Matomo HTTP at {url} ...")
|
||||
last_err: Exception | None = None
|
||||
|
||||
for i in range(timeout):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as resp:
|
||||
_ = resp.read(128)
|
||||
_log("[install] Matomo HTTP reachable (2xx/3xx).")
|
||||
return
|
||||
except urllib.error.HTTPError as exc:
|
||||
_log(f"[install] Matomo HTTP reachable (HTTP {exc.code}).")
|
||||
return
|
||||
except Exception as exc:
|
||||
last_err = exc
|
||||
if i % 5 == 0:
|
||||
_log(f"[install] still waiting ({i}/{timeout}) … ({type(exc).__name__})")
|
||||
time.sleep(1)
|
||||
|
||||
raise RuntimeError(f"Matomo did not become reachable after {timeout}s: {url} ({last_err})")
|
||||
|
||||
|
||||
def is_installed(url: str) -> bool:
|
||||
"""
|
||||
Heuristic:
|
||||
- installed instances typically render login module links
|
||||
- installer renders 'installation' wizard content
|
||||
"""
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=5) as resp:
|
||||
html = resp.read().decode(errors="ignore").lower()
|
||||
return ("module=login" in html) or ("matomo › login" in html) or ("matomo/login" in html)
|
||||
except urllib.error.HTTPError as exc:
|
||||
try:
|
||||
html = exc.read().decode(errors="ignore").lower()
|
||||
return ("module=login" in html) or ("matomo › login" in html) or ("matomo/login" in html)
|
||||
except Exception:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class WebInstaller(Installer):
|
||||
def ensure_installed(self, config: Config) -> None:
|
||||
"""
|
||||
Ensure Matomo is installed. NO-OP if already installed.
|
||||
Uses Playwright to drive the web installer (recorded flow).
|
||||
"""
|
||||
base_url = config.base_url
|
||||
|
||||
wait_http(base_url)
|
||||
|
||||
if is_installed(base_url):
|
||||
if config.debug:
|
||||
_log("[install] Matomo already looks installed. Skipping installer.")
|
||||
return
|
||||
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
_log("[install] Running Matomo web installer via Playwright (recorded flow)...")
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(
|
||||
headless=PLAYWRIGHT_HEADLESS,
|
||||
slow_mo=PLAYWRIGHT_SLOWMO_MS if PLAYWRIGHT_SLOWMO_MS > 0 else None,
|
||||
)
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
page.set_default_navigation_timeout(PLAYWRIGHT_NAV_TIMEOUT_MS)
|
||||
page.set_default_timeout(PLAYWRIGHT_NAV_TIMEOUT_MS)
|
||||
|
||||
def _dbg(msg: str) -> None:
|
||||
if config.debug:
|
||||
_log(f"[install] {msg}")
|
||||
|
||||
def click_next() -> None:
|
||||
"""
|
||||
Matomo installer mixes link/button variants and sometimes includes '»'.
|
||||
We try common variants in a robust order.
|
||||
"""
|
||||
candidates = [
|
||||
("link", "Next »"),
|
||||
("button", "Next »"),
|
||||
("link", "Next"),
|
||||
("button", "Next"),
|
||||
("link", "Continue"),
|
||||
("button", "Continue"),
|
||||
("link", "Proceed"),
|
||||
("button", "Proceed"),
|
||||
("link", "Start Installation"),
|
||||
("button", "Start Installation"),
|
||||
("link", "Weiter"),
|
||||
("button", "Weiter"),
|
||||
("link", "Fortfahren"),
|
||||
("button", "Fortfahren"),
|
||||
]
|
||||
|
||||
for role, name in candidates:
|
||||
loc = page.get_by_role(role, name=name)
|
||||
if loc.count() > 0:
|
||||
_dbg(f"click_next(): {role} '{name}'")
|
||||
loc.first.click()
|
||||
return
|
||||
|
||||
loc = page.get_by_text("Next", exact=False)
|
||||
if loc.count() > 0:
|
||||
_dbg("click_next(): fallback text 'Next'")
|
||||
loc.first.click()
|
||||
return
|
||||
|
||||
raise RuntimeError("Could not find a Next/Continue control in the installer UI.")
|
||||
|
||||
page.goto(base_url, wait_until="domcontentloaded")
|
||||
|
||||
def superuser_form_visible() -> bool:
|
||||
return page.locator("#login-0").count() > 0
|
||||
|
||||
for _ in range(12):
|
||||
if superuser_form_visible():
|
||||
break
|
||||
click_next()
|
||||
page.wait_for_load_state("domcontentloaded")
|
||||
page.wait_for_timeout(200)
|
||||
else:
|
||||
raise RuntimeError("Installer did not reach superuser step (login-0 not found).")
|
||||
|
||||
page.locator("#login-0").click()
|
||||
page.locator("#login-0").fill(config.admin_user)
|
||||
|
||||
page.locator("#password-0").click()
|
||||
page.locator("#password-0").fill(config.admin_password)
|
||||
|
||||
if page.locator("#password_bis-0").count() > 0:
|
||||
page.locator("#password_bis-0").click()
|
||||
page.locator("#password_bis-0").fill(config.admin_password)
|
||||
|
||||
page.locator("#email-0").click()
|
||||
page.locator("#email-0").fill(config.admin_email)
|
||||
|
||||
if page.get_by_role("button", name="Next »").count() > 0:
|
||||
page.get_by_role("button", name="Next »").click()
|
||||
else:
|
||||
click_next()
|
||||
|
||||
if page.locator("#siteName-0").count() > 0:
|
||||
page.locator("#siteName-0").click()
|
||||
page.locator("#siteName-0").fill(DEFAULT_SITE_NAME)
|
||||
|
||||
if page.locator("#url-0").count() > 0:
|
||||
page.locator("#url-0").click()
|
||||
page.locator("#url-0").fill(DEFAULT_SITE_URL)
|
||||
|
||||
try:
|
||||
page.get_by_role("combobox").first.click()
|
||||
page.get_by_role("listbox").get_by_text(DEFAULT_TIMEZONE).click()
|
||||
except Exception:
|
||||
_dbg("Timezone selection skipped (not found / changed UI).")
|
||||
|
||||
try:
|
||||
page.get_by_role("combobox").nth(2).click()
|
||||
page.get_by_role("listbox").get_by_text(DEFAULT_ECOMMERCE).click()
|
||||
except Exception:
|
||||
_dbg("Ecommerce selection skipped (not found / changed UI).")
|
||||
|
||||
click_next()
|
||||
page.wait_for_load_state("domcontentloaded")
|
||||
|
||||
if page.get_by_role("link", name="Next »").count() > 0:
|
||||
page.get_by_role("link", name="Next »").click()
|
||||
|
||||
if page.get_by_role("button", name="Continue to Matomo »").count() > 0:
|
||||
page.get_by_role("button", name="Continue to Matomo »").click()
|
||||
|
||||
context.close()
|
||||
browser.close()
|
||||
|
||||
time.sleep(1)
|
||||
if not is_installed(base_url):
|
||||
raise RuntimeError("[install] Installer did not reach installed state.")
|
||||
|
||||
_log("[install] Installation finished.")
|
||||
115
src/matomo_bootstrap/matomo_api.py
Normal file
115
src/matomo_bootstrap/matomo_api.py
Normal file
@@ -0,0 +1,115 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
|
||||
from .errors import MatomoNotReadyError, TokenCreationError
|
||||
from .http import HttpClient
|
||||
|
||||
|
||||
def _md5(text: str) -> str:
|
||||
return hashlib.md5(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _try_json(body: str) -> object:
|
||||
try:
|
||||
return json.loads(body)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise TokenCreationError(f"Invalid JSON from Matomo API: {body[:400]}") from exc
|
||||
|
||||
|
||||
def _dbg(msg: str, enabled: bool) -> None:
|
||||
if enabled:
|
||||
# Keep stdout clean (tests expect only token on stdout).
|
||||
print(msg, file=sys.stderr)
|
||||
|
||||
|
||||
class MatomoApi:
|
||||
def __init__(self, *, client: HttpClient, debug: bool = False):
|
||||
self.client = client
|
||||
self.debug = debug
|
||||
|
||||
def assert_ready(self, timeout: int = 10) -> None:
|
||||
"""
|
||||
Minimal readiness check: Matomo UI should be reachable and look like Matomo.
|
||||
"""
|
||||
try:
|
||||
status, body = self.client.get("/", {})
|
||||
except Exception as exc: # pragma: no cover
|
||||
raise MatomoNotReadyError(f"Matomo not reachable: {exc}") from exc
|
||||
|
||||
_dbg(f"[ready] GET / -> HTTP {status}", self.debug)
|
||||
|
||||
html = (body or "").lower()
|
||||
if "matomo" not in html and "piwik" not in html:
|
||||
raise MatomoNotReadyError("Matomo UI not detected at base URL")
|
||||
|
||||
def login_via_logme(self, admin_user: str, admin_password: str) -> None:
|
||||
"""
|
||||
Create an authenticated Matomo session (cookie jar) using Login controller.
|
||||
Matomo accepts md5 hashed password in `password` parameter for action=logme.
|
||||
"""
|
||||
md5_password = _md5(admin_password)
|
||||
try:
|
||||
status, body = self.client.get(
|
||||
"/index.php",
|
||||
{
|
||||
"module": "Login",
|
||||
"action": "logme",
|
||||
"login": admin_user,
|
||||
"password": md5_password,
|
||||
},
|
||||
)
|
||||
_dbg(f"[auth] logme HTTP {status} body[:120]={body[:120]!r}", self.debug)
|
||||
except urllib.error.HTTPError as exc:
|
||||
# Even 4xx/5xx can still set cookies; continue and let the API call validate.
|
||||
try:
|
||||
err_body = exc.read().decode("utf-8", errors="replace")
|
||||
except Exception:
|
||||
err_body = ""
|
||||
_dbg(f"[auth] logme HTTPError {exc.code} body[:120]={err_body[:120]!r}", self.debug)
|
||||
|
||||
def create_app_specific_token(
|
||||
self,
|
||||
*,
|
||||
admin_user: str,
|
||||
admin_password: str,
|
||||
description: str,
|
||||
) -> str:
|
||||
"""
|
||||
Create an app-specific token using an authenticated session (cookies),
|
||||
not UsersManager.getTokenAuth (not available in Matomo 5.3.x images).
|
||||
"""
|
||||
env_token = os.environ.get("MATOMO_BOOTSTRAP_TOKEN_AUTH")
|
||||
if env_token:
|
||||
_dbg("[auth] Using MATOMO_BOOTSTRAP_TOKEN_AUTH from environment.", self.debug)
|
||||
return env_token
|
||||
|
||||
self.login_via_logme(admin_user, admin_password)
|
||||
|
||||
status, body = self.client.post(
|
||||
"/index.php",
|
||||
{
|
||||
"module": "API",
|
||||
"method": "UsersManager.createAppSpecificTokenAuth",
|
||||
"userLogin": admin_user,
|
||||
"passwordConfirmation": admin_password,
|
||||
"description": description,
|
||||
"format": "json",
|
||||
},
|
||||
)
|
||||
|
||||
_dbg(f"[auth] createAppSpecificTokenAuth HTTP {status} body[:200]={body[:200]!r}", self.debug)
|
||||
|
||||
if status != 200:
|
||||
raise TokenCreationError(f"HTTP {status} during token creation: {body[:400]}")
|
||||
|
||||
data = _try_json(body)
|
||||
token = data.get("value") if isinstance(data, dict) else None
|
||||
if not token:
|
||||
raise TokenCreationError(f"Unexpected response from token creation: {data}")
|
||||
|
||||
return str(token)
|
||||
33
src/matomo_bootstrap/service.py
Normal file
33
src/matomo_bootstrap/service.py
Normal file
@@ -0,0 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .config import Config
|
||||
from .http import HttpClient
|
||||
from .matomo_api import MatomoApi
|
||||
from .installers.web import WebInstaller
|
||||
|
||||
|
||||
def run(config: Config) -> str:
|
||||
"""
|
||||
Orchestrate:
|
||||
1) Ensure Matomo is installed (NO-OP if installed)
|
||||
2) Ensure Matomo is reachable/ready
|
||||
3) Create an app-specific token using an authenticated session
|
||||
"""
|
||||
installer = WebInstaller()
|
||||
installer.ensure_installed(config)
|
||||
|
||||
client = HttpClient(
|
||||
base_url=config.base_url,
|
||||
timeout=config.timeout,
|
||||
debug=config.debug,
|
||||
)
|
||||
api = MatomoApi(client=client, debug=config.debug)
|
||||
|
||||
api.assert_ready(timeout=config.timeout)
|
||||
|
||||
token = api.create_app_specific_token(
|
||||
admin_user=config.admin_user,
|
||||
admin_password=config.admin_password,
|
||||
description=config.token_description,
|
||||
)
|
||||
return token
|
||||
@@ -1,6 +1,7 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
import urllib.request
|
||||
|
||||
@@ -14,7 +15,7 @@ ADMIN_EMAIL = os.environ.get("MATOMO_ADMIN_EMAIL", "administrator@example.org")
|
||||
class TestMatomoBootstrapE2E(unittest.TestCase):
|
||||
def test_bootstrap_creates_api_token(self) -> None:
|
||||
cmd = [
|
||||
"python3",
|
||||
sys.executable,
|
||||
"-m",
|
||||
"matomo_bootstrap",
|
||||
"--base-url",
|
||||
|
||||
Reference in New Issue
Block a user