kevinveenbirkenbach/matomo-bootstrap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(e2e): use Playwright web installer + session-based token creation for Matomo 5.3

Browse Source 
- make Playwright a runtime dependency (installer requires it)
- record-and-replay Matomo web installer flow (more stable than container bootstrap)
- replace removed UsersManager.getTokenAuth with cookie-session login + createAppSpecificTokenAuth
- make HttpClient return (status, body) for HTTPError responses
- set deterministic container names in docker-compose and pass MATOMO_CONTAINER_NAME

https://chatgpt.com/share/694a7b30-3dd4-800f-ba48-ae7083cfa4d8
This commit is contained in:
Kevin Veen-Birkenbach
2025-12-23 12:21:13 +01:00
parent da261e21e9
commit b63ea71902
8 changed files with 353 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@@ -63,6 +63,7 @@ e2e-install: playwright-install
MATOMO_ADMIN_PASSWORD="$(MATOMO_ADMIN_PASSWORD)" \
MATOMO_ADMIN_EMAIL="$(MATOMO_ADMIN_EMAIL)" \
MATOMO_TOKEN_DESCRIPTION="$(MATOMO_TOKEN_DESCRIPTION)" \
MATOMO_CONTAINER_NAME="e2e-matomo" \
PYTHONPATH=src $(VENV_PY) -m matomo_bootstrap
e2e-test: deps-e2e

9
pyproject.toml
View File

@@ -12,13 +12,14 @@ authors = [{ name = "Kevin Veen-Birkenbach", email = "kevin@veen.world" }]
license = { text = "All rights reserved by Kevin Veen-Birkenbach" }
urls = { Homepage = "https://github.com/kevinveenbirkenbach/matomo-bootstrap" }
dependencies = []
[project.optional-dependencies]
e2e = [
# Playwright is needed at runtime to run the web installer when Matomo is not yet installed.
dependencies = [
"playwright>=1.40.0",
]
[project.optional-dependencies]
e2e = []
[tool.setuptools]
package-dir = { "" = "src" }

111
src/matomo_bootstrap/api_tokens.py
View File

@@ -1,56 +1,83 @@
import hashlib
import json
import os
import urllib.error
from .errors import TokenCreationError
from .http import HttpClient
def get_token_auth(client: HttpClient, admin_user: str, admin_password: str) -> str:
"""
Get the user's token_auth via UsersManager.getTokenAuth.
def _md5(text: str) -> str:
return hashlib.md5(text.encode("utf-8")).hexdigest()
This is the most robust way to authenticate subsequent API calls without relying
on UI sessions/cookies.
"""
md5_password = hashlib.md5(admin_password.encode("utf-8")).hexdigest()
status, body = client.get(
"/index.php",
{
"module": "API",
"method": "UsersManager.getTokenAuth",
"userLogin": admin_user,
"md5Password": md5_password,
"format": "json",
},
)
if status != 200:
raise TokenCreationError(f"HTTP {status} during getTokenAuth: {body[:200]}")
def _try_json(body: str) -> object:
try:
data = json.loads(body)
return json.loads(body)
except json.JSONDecodeError as exc:
raise TokenCreationError(f"Invalid JSON from getTokenAuth: {body[:200]}") from exc
# Matomo returns either {"value": "..."} or sometimes a plain string depending on setup/version
if isinstance(data, dict) and data.get("value"):
return str(data["value"])
if isinstance(data, str) and data:
return data
raise TokenCreationError(f"Unexpected getTokenAuth response: {data}")
raise TokenCreationError(f"Invalid JSON from Matomo API: {body[:400]}") from exc
def create_app_token(
def _login_via_logme(client: HttpClient, admin_user: str, admin_password: str, debug: bool) -> None:
"""
Create an authenticated Matomo session (cookie jar) using the classic Login controller.
Matomo accepts the md5 hashed password in the `password` parameter for action=logme.
We rely on urllib's opener to follow redirects and store cookies.
If this ever stops working in a future Matomo version, the next step would be:
- GET the login page, extract CSRF/nonce, then POST the login form.
"""
md5_password = _md5(admin_password)
# Hit the login endpoint; cookies should be set in the client's CookieJar.
# We treat any HTTP response as "we reached the login controller" – later API call will tell us if session is valid.
try:
status, body = client.get(
"/index.php",
{
"module": "Login",
"action": "logme",
"login": admin_user,
"password": md5_password,
},
)
if debug:
print(f"[auth] login via logme returned HTTP {status} (body preview: {body[:120]!r})")
except urllib.error.HTTPError as exc:
# Even 4xx/5xx can still set cookies; continue and let the API call validate.
if debug:
try:
err_body = exc.read().decode("utf-8", errors="replace")
except Exception:
err_body = ""
print(f"[auth] login via logme raised HTTPError {exc.code} (body preview: {err_body[:120]!r})")
def create_app_token_via_session(
*,
client: HttpClient,
admin_token_auth: str,
admin_user: str,
admin_password: str,
description: str,
debug: bool = False,
) -> str:
"""
Create an app-specific token using token_auth authentication.
Create an app-specific token using an authenticated SESSION (cookies),
not via UsersManager.getTokenAuth (removed/not available in Matomo 5.3.x images).
If MATOMO_BOOTSTRAP_TOKEN_AUTH is already set, we return it.
"""
env_token = os.environ.get("MATOMO_BOOTSTRAP_TOKEN_AUTH")
if env_token:
if debug:
print("[auth] Using MATOMO_BOOTSTRAP_TOKEN_AUTH from environment.")
return env_token
# 1) Establish logged-in session
_login_via_logme(client, admin_user, admin_password, debug=debug)
# 2) Use the session cookie to create an app specific token
status, body = client.post(
"/index.php",
{
@@ -60,20 +87,20 @@ def create_app_token(
"passwordConfirmation": admin_password,
"description": description,
"format": "json",
"token_auth": admin_token_auth,
},
)
if status != 200:
raise TokenCreationError(f"HTTP {status} during token creation: {body[:200]}")
if debug:
print(f"[auth] createAppSpecificTokenAuth HTTP {status} body[:200]={body[:200]!r}")
try:
data = json.loads(body)
except json.JSONDecodeError as exc:
raise TokenCreationError(f"Invalid JSON from Matomo API: {body[:200]}") from exc
if status != 200:
raise TokenCreationError(f"HTTP {status} during token creation: {body[:400]}")
data = _try_json(body)
token = data.get("value") if isinstance(data, dict) else None
if not token:
raise TokenCreationError(f"Unexpected response: {data}")
# Matomo may return {"result":"error","message":"..."}.
raise TokenCreationError(f"Unexpected response from token creation: {data}")
return str(token)

14
src/matomo_bootstrap/bootstrap.py
View File

@@ -1,6 +1,6 @@
from argparse import Namespace
from .api_tokens import create_app_token, get_token_auth
from .api_tokens import create_app_token_via_session
from .health import assert_matomo_ready
from .http import HttpClient
from .install.web_installer import ensure_installed
@@ -19,25 +19,19 @@ def run_bootstrap(args: Namespace) -> str:
# 2) Now the UI/API should be reachable and "installed"
assert_matomo_ready(args.base_url, timeout=args.timeout)
# 3) Create authenticated API token flow (no UI session needed)
# 3) Create app-specific token via authenticated session (cookie-based)
client = HttpClient(
base_url=args.base_url,
timeout=args.timeout,
debug=args.debug,
)
admin_token_auth = get_token_auth(
token = create_app_token_via_session(
client=client,
admin_user=args.admin_user,
admin_password=args.admin_password,
)
token = create_app_token(
client=client,
admin_token_auth=admin_token_auth,
admin_user=args.admin_user,
admin_password=args.admin_password,
description=args.token_description,
debug=args.debug,
)
return token

30
src/matomo_bootstrap/http.py
View File

@@ -1,4 +1,5 @@
import http.cookiejar
import urllib.error
import urllib.parse
import urllib.request
from typing import Dict, Tuple
@@ -23,9 +24,18 @@ class HttpClient:
print(f"[HTTP] GET {url}")
req = urllib.request.Request(url, method="GET")
with self.opener.open(req, timeout=self.timeout) as resp:
body = resp.read().decode("utf-8", errors="replace")
return resp.status, body
try:
with self.opener.open(req, timeout=self.timeout) as resp:
body = resp.read().decode("utf-8", errors="replace")
return resp.status, body
except urllib.error.HTTPError as exc:
# urllib raises HTTPError for 4xx/5xx but it still contains status + body
try:
body = exc.read().decode("utf-8", errors="replace")
except Exception:
body = str(exc)
return exc.code, body
def post(self, path: str, data: Dict[str, str]) -> Tuple[int, str]:
url = self.base_url + path
@@ -35,6 +45,14 @@ class HttpClient:
print(f"[HTTP] POST {url} keys={list(data.keys())}")
req = urllib.request.Request(url, data=encoded, method="POST")
with self.opener.open(req, timeout=self.timeout) as resp:
body = resp.read().decode("utf-8", errors="replace")
return resp.status, body
try:
with self.opener.open(req, timeout=self.timeout) as resp:
body = resp.read().decode("utf-8", errors="replace")
return resp.status, body
except urllib.error.HTTPError as exc:
try:
body = exc.read().decode("utf-8", errors="replace")
except Exception:
body = str(exc)
return exc.code, body

125
src/matomo_bootstrap/install/container_installer.py
View File

@@ -1,34 +1,113 @@
import subprocess
import time
from typing import Optional
def ensure_installed(
container_name: str = "e2e-matomo-1",
MATOMO_ROOT = "/var/www/html"
CONSOLE = f"{MATOMO_ROOT}/console"
def _run(cmd: list[str]) -> subprocess.CompletedProcess:
return subprocess.run(cmd, text=True, capture_output=True)
def _container_state(container_name: str) -> str:
res = _run(["docker", "inspect", "-f", "{{.State.Status}}", container_name])
return (res.stdout or "").strip()
def _wait_container_running(container_name: str, timeout: int = 60) -> None:
last = ""
for _ in range(timeout):
state = _container_state(container_name)
last = state
if state == "running":
return
time.sleep(1)
raise RuntimeError(f"Container '{container_name}' did not become running (last state: {last})")
def _exec(container_name: str, argv: list[str]) -> subprocess.CompletedProcess:
return _run(["docker", "exec", container_name, *argv])
def _sh(container_name: str, script: str) -> subprocess.CompletedProcess:
# Use sh -lc so PATH + cwd behave more like interactive container sessions
return _exec(container_name, ["sh", "-lc", script])
def _console_exists(container_name: str) -> bool:
res = _sh(container_name, f"test -x {CONSOLE} && echo yes || echo no")
return (res.stdout or "").strip() == "yes"
def _is_installed(container_name: str) -> bool:
res = _sh(container_name, f"test -f {MATOMO_ROOT}/config/config.ini.php && echo yes || echo no")
return (res.stdout or "").strip() == "yes"
def _console_list(container_name: str) -> str:
# --no-ansi for stable parsing
res = _sh(container_name, f"{CONSOLE} list --no-ansi 2>/dev/null || true")
return (res.stdout or "") + "\n" + (res.stderr or "")
def _has_command(console_list_output: str, command: str) -> bool:
# cheap but robust enough
return f" {command} " in console_list_output or f"\n{command}\n" in console_list_output or command in console_list_output
def ensure_installed_via_console(
*,
container_name: str,
admin_user: str,
admin_password: str,
admin_email: str,
debug: bool = False,
) -> None:
"""
Ensure Matomo is installed by executing PHP bootstrap inside container.
Idempotent: safe to run multiple times.
Ensure Matomo is installed using the container's console if possible.
If no known install command exists, we do NOT guess: we raise with diagnostics.
"""
_wait_container_running(container_name, timeout=90)
cmd = [
"docker",
"exec",
container_name,
"php",
"-r",
r"""
if (file_exists('/var/www/html/config/config.ini.php')) {
echo "Matomo already installed\n";
exit(0);
}
if _is_installed(container_name):
if debug:
print("[install] Matomo already installed (config.ini.php exists).")
return
require '/var/www/html/core/bootstrap.php';
if not _console_exists(container_name):
raise RuntimeError(f"Matomo console not found/executable at {CONSOLE} inside container '{container_name}'.")
\Piwik\FrontController::getInstance()->init();
\Piwik\Plugins\Installation\Installation::install();
listing = _console_list(container_name)
if debug:
print("[install] Matomo console list obtained.")
echo "Matomo installed\n";
"""
]
subprocess.check_call(cmd)
# Matomo versions differ; we discover what exists.
# Historically: core:install. Your earlier log showed it does NOT exist in 5.3.2 image.
# Therefore we refuse to guess and provide the list in the exception.
if _has_command(listing, "core:install"):
# If this ever exists, use it.
cmd = (
f"{CONSOLE} core:install --no-ansi "
f"--database-host=db "
f"--database-username=matomo "
f"--database-password=matomo_pw "
f"--database-name=matomo "
f"--login={admin_user} "
f"--password={admin_password} "
f"--email={admin_email} "
f"--url=http://localhost "
)
res = _sh(container_name, cmd)
if res.returncode != 0:
raise RuntimeError(f"Matomo CLI install failed.\nexit={res.returncode}\nstdout:\n{res.stdout}\nstderr:\n{res.stderr}")
return
# No install command -> fail with diagnostics (don’t keep burning time).
raise RuntimeError(
"Matomo is not installed yet, but no supported CLI install command was found in this image.\n"
"This Matomo image likely expects the web installer.\n"
"\n[console list]\n"
f"{listing}\n"
)

225
src/matomo_bootstrap/install/web_installer.py
View File

@@ -1,15 +1,19 @@
import os
import sys
import time
import urllib.error
import urllib.request
DB_HOST = os.environ.get("MATOMO_DB_HOST", "db")
DB_USER = os.environ.get("MATOMO_DB_USER", "matomo")
DB_PASS = os.environ.get("MATOMO_DB_PASS", "matomo_pw")
DB_NAME = os.environ.get("MATOMO_DB_NAME", "matomo")
DB_PREFIX = os.environ.get("MATOMO_DB_PREFIX", "matomo_")
# Optional knobs (mostly for debugging / CI stability)
PLAYWRIGHT_HEADLESS = os.environ.get("MATOMO_PLAYWRIGHT_HEADLESS", "1").strip() not in ("0", "false", "False")
PLAYWRIGHT_SLOWMO_MS = int(os.environ.get("MATOMO_PLAYWRIGHT_SLOWMO_MS", "0"))
PLAYWRIGHT_NAV_TIMEOUT_MS = int(os.environ.get("MATOMO_PLAYWRIGHT_NAV_TIMEOUT_MS", "60000"))
# Values used by the installer flow (recorded)
DEFAULT_SITE_NAME = os.environ.get("MATOMO_SITE_NAME", "localhost")
DEFAULT_SITE_URL = os.environ.get("MATOMO_SITE_URL", "http://localhost")
DEFAULT_TIMEZONE = os.environ.get("MATOMO_TIMEZONE", "Germany - Berlin")
DEFAULT_ECOMMERCE = os.environ.get("MATOMO_ECOMMERCE", "Ecommerce enabled")
def wait_http(url: str, timeout: int = 180) -> None:
@@ -27,7 +31,6 @@ def wait_http(url: str, timeout: int = 180) -> None:
print("[install] Matomo HTTP reachable (2xx/3xx).")
return
except urllib.error.HTTPError as exc:
# 4xx/5xx means the server answered -> reachable
print(f"[install] Matomo HTTP reachable (HTTP {exc.code}).")
return
except Exception as exc:
@@ -50,7 +53,6 @@ def is_installed(url: str) -> bool:
html = resp.read().decode(errors="ignore").lower()
return ("module=login" in html) or ("matomo › login" in html) or ("matomo/login" in html)
except urllib.error.HTTPError as exc:
# Even if it's 500, read body and try heuristic.
try:
html = exc.read().decode(errors="ignore").lower()
return ("module=login" in html) or ("matomo › login" in html) or ("matomo/login" in html)
@@ -70,101 +72,166 @@ def ensure_installed(
"""
Ensure Matomo is installed.
NO-OP if already installed.
This implementation ONLY uses the Playwright web installer (recorded flow).
"""
wait_http(base_url)
if is_installed(base_url):
if debug:
print("[install] Matomo already looks installed. Skipping web installer.")
print("[install] Matomo already looks installed. Skipping installer.")
return
try:
from playwright.sync_api import sync_playwright
except Exception as exc:
print("[install] Playwright not available.", file=sys.stderr)
print(
"Install with: python3 -m pip install playwright && python3 -m playwright install chromium",
file=sys.stderr,
)
raise RuntimeError(f"Playwright missing: {exc}") from exc
from playwright.sync_api import sync_playwright
print("[install] Running Matomo web installer via headless browser...")
print("[install] Running Matomo web installer via Playwright (recorded flow)...")
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
page = browser.new_page()
browser = p.chromium.launch(
headless=PLAYWRIGHT_HEADLESS,
slow_mo=PLAYWRIGHT_SLOWMO_MS if PLAYWRIGHT_SLOWMO_MS > 0 else None,
)
context = browser.new_context()
page = context.new_page()
page.set_default_navigation_timeout(PLAYWRIGHT_NAV_TIMEOUT_MS)
page.set_default_timeout(PLAYWRIGHT_NAV_TIMEOUT_MS)
# Load installer (may be 500 in curl, but browser can still render the Matomo error/installer flow)
page.goto(base_url, wait_until="domcontentloaded")
def _dbg(msg: str) -> None:
if debug:
print(f"[install] {msg}")
def click_next() -> None:
# Buttons vary slightly with locales/versions
for label in ["Next", "Continue", "Start Installation", "Proceed", "Weiter", "Fortfahren"]:
btn = page.get_by_role("button", name=label)
if btn.count() > 0:
btn.first.click()
"""
Matomo installer mixes link/button variants and sometimes includes '»'.
We try common variants in a robust order.
"""
candidates = [
("link", "Next »"),
("button", "Next »"),
("link", "Next"),
("button", "Next"),
("link", "Continue"),
("button", "Continue"),
("link", "Proceed"),
("button", "Proceed"),
("link", "Start Installation"),
("button", "Start Installation"),
("link", "Weiter"),
("button", "Weiter"),
("link", "Fortfahren"),
("button", "Fortfahren"),
]
for role, name in candidates:
loc = page.get_by_role(role, name=name)
if loc.count() > 0:
_dbg(f"click_next(): {role} '{name}'")
loc.first.click()
return
# Sometimes it's a link styled as button
for text in ["Next", "Continue", "Start Installation", "Proceed", "Weiter", "Fortfahren"]:
a = page.get_by_text(text, exact=False)
if a.count() > 0:
a.first.click()
return
raise RuntimeError("Could not find a 'Next/Continue' control in installer UI.")
# Welcome / System check
page.wait_for_timeout(700)
click_next()
page.wait_for_timeout(700)
click_next()
# last resort: some pages use same text but different element types
loc = page.get_by_text("Next", exact=False)
if loc.count() > 0:
_dbg("click_next(): fallback text 'Next'")
loc.first.click()
return
# Database setup
page.wait_for_timeout(700)
page.get_by_label("Database Server").fill(DB_HOST)
page.get_by_label("Login").fill(DB_USER)
page.get_by_label("Password").fill(DB_PASS)
page.get_by_label("Database Name").fill(DB_NAME)
try:
page.get_by_label("Tables Prefix").fill(DB_PREFIX)
except Exception:
pass
click_next()
raise RuntimeError("Could not find a Next/Continue control in the installer UI.")
# Tables creation
page.wait_for_timeout(700)
click_next()
# --- Recorded-ish flow, but made variable-based + more stable ---
page.goto(base_url, wait_until="domcontentloaded")
# Super user
page.wait_for_timeout(700)
page.get_by_label("Login").fill(admin_user)
page.get_by_label("Password").fill(admin_password)
try:
page.get_by_label("Password (repeat)").fill(admin_password)
except Exception:
pass
page.get_by_label("Email").fill(admin_email)
click_next()
# The first few screens can vary slightly (welcome/system check/db etc.).
# In your recording, you clicked through multiple Next pages without DB input (env already set in container).
# We mimic that: keep clicking "Next" until we see the superuser fields.
#
# Stop condition: superuser login field appears.
def superuser_form_visible() -> bool:
# In your recording, the superuser "Login" field was "#login-0".
return page.locator("#login-0").count() > 0
# Click next until the superuser page shows up (cap to avoid infinite loops).
for _ in range(12):
if superuser_form_visible():
break
click_next()
page.wait_for_load_state("domcontentloaded")
page.wait_for_timeout(200)
else:
raise RuntimeError("Installer did not reach superuser step (login-0 not found).")
# Superuser step
page.locator("#login-0").click()
page.locator("#login-0").fill(admin_user)
page.locator("#password-0").click()
page.locator("#password-0").fill(admin_password)
# Repeat password (some versions have it)
if page.locator("#password_bis-0").count() > 0:
page.locator("#password_bis-0").click()
page.locator("#password_bis-0").fill(admin_password)
page.locator("#email-0").click()
page.locator("#email-0").fill(admin_email)
# Next
page.get_by_role("button", name="Next »").click()
# First website
page.wait_for_timeout(700)
try:
page.get_by_label("Name").fill("Bootstrap Site")
except Exception:
pass
try:
page.get_by_label("URL").fill("http://example.invalid")
except Exception:
pass
click_next()
if page.locator("#siteName-0").count() > 0:
page.locator("#siteName-0").click()
page.locator("#siteName-0").fill(DEFAULT_SITE_NAME)
# Finish
page.wait_for_timeout(700)
click_next()
if page.locator("#url-0").count() > 0:
page.locator("#url-0").click()
page.locator("#url-0").fill(DEFAULT_SITE_URL)
# Timezone dropdown (best-effort)
try:
# recording: page.get_by_role("combobox").first.click() then listbox text
page.get_by_role("combobox").first.click()
page.get_by_role("listbox").get_by_text(DEFAULT_TIMEZONE).click()
except Exception:
_dbg("Timezone selection skipped (not found / changed UI).")
# Ecommerce dropdown (best-effort)
try:
# recording: combobox nth(2)
page.get_by_role("combobox").nth(2).click()
page.get_by_role("listbox").get_by_text(DEFAULT_ECOMMERCE).click()
except Exception:
_dbg("Ecommerce selection skipped (not found / changed UI).")
# Next pages to finish
click_next()
page.wait_for_load_state("domcontentloaded")
# In recording: Next link, then Continue to Matomo button
if page.get_by_role("link", name="Next »").count() > 0:
page.get_by_role("link", name="Next »").click()
if page.get_by_role("button", name="Continue to Matomo »").count() > 0:
page.get_by_role("button", name="Continue to Matomo »").click()
# Optional: login once (not strictly required for token flow, but harmless and matches your recording).
# Some UIs have fancy-icon labels; we follow your recorded selectors best-effort.
try:
user_box = page.get_by_role("textbox", name=" Username or e-mail")
pass_box = page.get_by_role("textbox", name=" Password")
if user_box.count() > 0 and pass_box.count() > 0:
user_box.click()
user_box.fill(admin_user)
pass_box.fill(admin_password)
if page.get_by_role("button", name="Sign in").count() > 0:
page.get_by_role("button", name="Sign in").click()
except Exception:
_dbg("Post-install login skipped (UI differs).")
context.close()
browser.close()
# Verify installed
time.sleep(2)
time.sleep(1)
if not is_installed(base_url):
raise RuntimeError("[install] Installer did not reach installed state.")

2
tests/e2e/docker-compose.yml
View File

@@ -1,6 +1,7 @@
services:
db:
image: mariadb:11
container_name: e2e-db
environment:
MARIADB_DATABASE: matomo
MARIADB_USER: matomo
@@ -14,6 +15,7 @@ services:
matomo:
image: matomo:5.3.2
container_name: e2e-matomo
depends_on:
db:
condition: service_healthy