diff --git a/pyproject.toml b/pyproject.toml index 11320e8..b9510f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ description = "Headless bootstrap tooling for Matomo (installation + API token p readme = "README.md" requires-python = ">=3.10" authors = [{ name = "Kevin Veen-Birkenbach", email = "kevin@veen.world" }] -license = { text = "MIT" } +license = "MIT" urls = { Homepage = "https://github.com/kevinveenbirkenbach/matomo-bootstrap" } dependencies = ["playwright>=1.46.0,<2"] diff --git a/src/matomo_bootstrap/installers/web.py b/src/matomo_bootstrap/installers/web.py index 5181574..15da0cd 100644 --- a/src/matomo_bootstrap/installers/web.py +++ b/src/matomo_bootstrap/installers/web.py @@ -28,6 +28,12 @@ INSTALLER_STEP_TIMEOUT_S = int(os.environ.get("MATOMO_INSTALLER_STEP_TIMEOUT_S", INSTALLER_STEP_DEADLINE_S = int( os.environ.get("MATOMO_INSTALLER_STEP_DEADLINE_S", "180") ) +INSTALLER_TABLES_CREATION_TIMEOUT_S = int( + os.environ.get("MATOMO_INSTALLER_TABLES_CREATION_TIMEOUT_S", "180") +) +INSTALLER_TABLES_ERASE_TIMEOUT_S = int( + os.environ.get("MATOMO_INSTALLER_TABLES_ERASE_TIMEOUT_S", "120") +) INSTALLER_DEBUG_DIR = os.environ.get( "MATOMO_INSTALLER_DEBUG_DIR", "/tmp/matomo-bootstrap" ).rstrip("/") @@ -280,12 +286,13 @@ def _wait_for_installer_interactive(page, *, timeout_s: int) -> None: def _click_next_with_wait(page, *, timeout_s: int) -> str: + before_url = page.url + before_step = _get_step_hint(before_url) + last_warning_log_at = 0.0 deadline = time.time() + timeout_s while time.time() < deadline: loc, label = _first_next_locator(page) if loc is not None: - before_url = page.url - before_step = _get_step_hint(before_url) try: loc.click(timeout=2_000) except Exception: @@ -299,6 +306,23 @@ def _click_next_with_wait(page, *, timeout_s: int) -> str: f"(url {before_url} -> {after_url})" ) return after_step + + _wait_dom_settled(page) + current_url = page.url + current_step = _get_step_hint(current_url) + if current_url != before_url or current_step != before_step: + _log( + "[install] Installer progressed without explicit click; " + f"step {before_step} -> {current_step} " + f"(url {before_url} -> {current_url})" + ) + return current_step + + now = time.time() + if now - last_warning_log_at >= 5: + _page_warnings(page) + last_warning_log_at = now + page.wait_for_timeout(300) raise RuntimeError( @@ -307,6 +331,139 @@ def _click_next_with_wait(page, *, timeout_s: int) -> str: ) +def _first_erase_tables_locator(page): + css_loc = page.locator("#eraseAllTables") + try: + if css_loc.count() > 0: + return css_loc.first, "css:#eraseAllTables" + except Exception: + pass + + for role, name in [ + ("link", "Delete the detected tables »"), + ("button", "Delete the detected tables »"), + ("link", "Delete the detected tables"), + ("button", "Delete the detected tables"), + ]: + loc = page.get_by_role(role, name=name) + try: + if loc.count() > 0: + return loc.first, f"{role}:{name}" + except Exception: + continue + + text_loc = page.get_by_text("Delete the detected tables", exact=False) + try: + if text_loc.count() > 0: + return text_loc.first, "text:Delete the detected tables*" + except Exception: + pass + + return None, "" + + +def _resolve_tables_creation_conflict(page, *, timeout_s: int) -> bool: + before_url = page.url + before_step = _get_step_hint(before_url) + if "tablesCreation" not in before_step: + return False + + loc, label = _first_erase_tables_locator(page) + if loc is None: + return False + + _log( + "[install] Detected existing tables during tablesCreation. " + f"Trying cleanup via {label}." + ) + + def _cleanup_url() -> str | None: + try: + href = page.locator("#eraseAllTables").first.get_attribute("href") + if href: + return urllib.parse.urljoin(page.url, href) + except Exception: + pass + + try: + parsed = urllib.parse.urlparse(page.url) + qs = urllib.parse.parse_qs(parsed.query, keep_blank_values=True) + if (qs.get("action") or [""])[0] != "tablesCreation": + return None + qs["deleteTables"] = ["1"] + return urllib.parse.urlunparse( + parsed._replace(query=urllib.parse.urlencode(qs, doseq=True)) + ) + except Exception: + return None + + deadline = time.time() + timeout_s + while time.time() < deadline: + accepted_dialog = False + + def _accept_dialog(dialog) -> None: + nonlocal accepted_dialog + accepted_dialog = True + try: + _log(f"[install] Accepting installer dialog: {dialog.message}") + except Exception: + _log("[install] Accepting installer dialog.") + try: + dialog.accept() + except Exception: + pass + + page.on("dialog", _accept_dialog) + try: + loc.click(timeout=2_000, force=True) + _wait_dom_settled(page) + except Exception as exc: + _log(f"[install] Cleanup click via {label} failed: {exc}") + cleanup_url = _cleanup_url() + if cleanup_url: + try: + page.goto(cleanup_url, wait_until="domcontentloaded") + _wait_dom_settled(page) + _log( + "[install] Triggered existing-table cleanup via URL fallback: " + f"{cleanup_url}" + ) + except Exception as nav_exc: + _log( + "[install] Cleanup URL fallback failed: " + f"{cleanup_url} ({nav_exc})" + ) + finally: + page.remove_listener("dialog", _accept_dialog) + + if accepted_dialog: + _log("[install] Existing-table cleanup dialog accepted.") + + _wait_dom_settled(page) + current_url = page.url + current_step = _get_step_hint(current_url) + if current_url != before_url or current_step != before_step: + _log( + "[install] Existing-table cleanup progressed installer; " + f"step {before_step} -> {current_step} " + f"(url {before_url} -> {current_url})" + ) + return True + + remaining_loc, _ = _first_erase_tables_locator(page) + if remaining_loc is None: + _log("[install] Existing-table cleanup control is gone.") + return True + + loc = remaining_loc + page.wait_for_timeout(500) + + raise RuntimeError( + "Detected existing Matomo tables but cleanup did not complete " + f"within {timeout_s}s (url={page.url}, step={_get_step_hint(page.url)})." + ) + + def wait_http(url: str, timeout: int = 180) -> None: """ Consider Matomo 'reachable' as soon as the HTTP server answers - even with 500. @@ -409,7 +566,17 @@ class WebInstaller(Installer): f"within {INSTALLER_STEP_DEADLINE_S}s " f"(url={page.url}, step={_get_step_hint(page.url)})." ) - _click_next_with_wait(page, timeout_s=INSTALLER_STEP_TIMEOUT_S) + if _resolve_tables_creation_conflict( + page, timeout_s=INSTALLER_TABLES_ERASE_TIMEOUT_S + ): + _page_warnings(page) + continue + step_timeout = INSTALLER_STEP_TIMEOUT_S + if "tablesCreation" in _get_step_hint(page.url): + step_timeout = max( + step_timeout, INSTALLER_TABLES_CREATION_TIMEOUT_S + ) + _click_next_with_wait(page, timeout_s=step_timeout) _page_warnings(page) page.locator("#login-0").click() @@ -426,34 +593,181 @@ class WebInstaller(Installer): page.locator("#email-0").fill(config.admin_email) _page_warnings(page) - _click_next_with_wait(page, timeout_s=INSTALLER_STEP_TIMEOUT_S) + submitted_superuser = False + try: + submitted_superuser = bool( + page.evaluate( + """ + ([user, password, email]) => { + const form = document.querySelector("form#generalsetupform"); + if (!form) return false; + + const loginInput = form.querySelector("input[name='login']"); + const passwordInput = form.querySelector("input[name='password']"); + const repeatPasswordInput = form.querySelector("input[name='password_bis']"); + const emailInput = form.querySelector("input[name='email']"); + if (!loginInput || !passwordInput || !emailInput) return false; + + loginInput.value = user; + passwordInput.value = password; + if (repeatPasswordInput) { + repeatPasswordInput.value = password; + } + emailInput.value = email; + + if (typeof form.requestSubmit === "function") { + form.requestSubmit(); + } else { + form.submit(); + } + return true; + } + """, + [ + config.admin_user, + config.admin_password, + config.admin_email, + ], + ) + ) + except Exception: + submitted_superuser = False + + if submitted_superuser: + _wait_dom_settled(page) + _log("[install] Submitted superuser form via form.requestSubmit().") + elif page.locator("#submit-0").count() > 0: + page.locator("#submit-0").click(timeout=2_000) + _wait_dom_settled(page) + _log("[install] Submitted superuser form via #submit-0 fallback.") + else: + _click_next_with_wait(page, timeout_s=INSTALLER_STEP_TIMEOUT_S) + + superuser_progress_deadline = time.time() + INSTALLER_STEP_TIMEOUT_S + while time.time() < superuser_progress_deadline: + _wait_dom_settled(page) + if page.locator("#login-0").count() == 0: + break + page.wait_for_timeout(300) + if page.locator("#login-0").count() > 0: + _page_warnings(page) + raise RuntimeError( + "Superuser form submit did not progress to first website setup " + f"within {INSTALLER_STEP_TIMEOUT_S}s " + f"(url={page.url}, step={_get_step_hint(page.url)})." + ) + _page_warnings(page) + submitted_first_website = False + try: + submitted_first_website = bool( + page.evaluate( + """ + ([siteName, siteUrl, timezoneLabel, ecommerceLabel]) => { + const form = document.querySelector("form#websitesetupform"); + if (!form) return false; + + const siteNameInput = form.querySelector("input[name='siteName']"); + const siteUrlInput = form.querySelector("input[name='url']"); + if (!siteNameInput || !siteUrlInput) return false; + + siteNameInput.value = siteName; + siteUrlInput.value = siteUrl; + + const timezoneSelect = form.querySelector("select[name='timezone']"); + if (timezoneSelect) { + const timezoneOption = Array.from(timezoneSelect.options).find( + (opt) => (opt.textContent || "").trim() === timezoneLabel + ); + if (timezoneOption) { + timezoneSelect.value = timezoneOption.value; + } + } + + const ecommerceSelect = form.querySelector("select[name='ecommerce']"); + if (ecommerceSelect) { + const ecommerceOption = Array.from(ecommerceSelect.options).find( + (opt) => (opt.textContent || "").trim() === ecommerceLabel + ); + if (ecommerceOption) { + ecommerceSelect.value = ecommerceOption.value; + } + } + + if (typeof form.requestSubmit === "function") { + form.requestSubmit(); + } else { + form.submit(); + } + return true; + } + """, + [ + DEFAULT_SITE_NAME, + DEFAULT_SITE_URL, + DEFAULT_TIMEZONE, + DEFAULT_ECOMMERCE, + ], + ) + ) + except Exception: + submitted_first_website = False + + if submitted_first_website: + _wait_dom_settled(page) + _log( + "[install] Submitted first website form via form.requestSubmit()." + ) + else: + if page.locator("#siteName-0").count() > 0: + page.locator("#siteName-0").click() + page.locator("#siteName-0").fill(DEFAULT_SITE_NAME) + + if page.locator("#url-0").count() > 0: + page.locator("#url-0").click() + page.locator("#url-0").fill(DEFAULT_SITE_URL) + + _page_warnings(page) + + try: + comboboxes = page.get_by_role("combobox") + if comboboxes.count() > 0: + comboboxes.first.click(timeout=2_000) + page.get_by_role("listbox").get_by_text( + DEFAULT_TIMEZONE + ).click(timeout=2_000) + except Exception: + _log("Timezone selection skipped (not found / changed UI).") + + try: + comboboxes = page.get_by_role("combobox") + if comboboxes.count() > 2: + comboboxes.nth(2).click(timeout=2_000) + page.get_by_role("listbox").get_by_text( + DEFAULT_ECOMMERCE + ).click(timeout=2_000) + except Exception: + _log("Ecommerce selection skipped (not found / changed UI).") + + _page_warnings(page) + + _click_next_with_wait(page, timeout_s=INSTALLER_STEP_TIMEOUT_S) + + first_website_progress_deadline = time.time() + INSTALLER_STEP_TIMEOUT_S + while time.time() < first_website_progress_deadline: + _wait_dom_settled(page) + if page.locator("#siteName-0").count() == 0: + break + page.wait_for_timeout(300) if page.locator("#siteName-0").count() > 0: - page.locator("#siteName-0").click() - page.locator("#siteName-0").fill(DEFAULT_SITE_NAME) + _page_warnings(page) + raise RuntimeError( + "First website form submit did not progress to tracking code " + f"within {INSTALLER_STEP_TIMEOUT_S}s " + f"(url={page.url}, step={_get_step_hint(page.url)})." + ) - if page.locator("#url-0").count() > 0: - page.locator("#url-0").click() - page.locator("#url-0").fill(DEFAULT_SITE_URL) - - _page_warnings(page) - - try: - page.get_by_role("combobox").first.click() - page.get_by_role("listbox").get_by_text(DEFAULT_TIMEZONE).click() - except Exception: - _log("Timezone selection skipped (not found / changed UI).") - - try: - page.get_by_role("combobox").nth(2).click() - page.get_by_role("listbox").get_by_text(DEFAULT_ECOMMERCE).click() - except Exception: - _log("Ecommerce selection skipped (not found / changed UI).") - - _page_warnings(page) - - _click_next_with_wait(page, timeout_s=INSTALLER_STEP_TIMEOUT_S) _page_warnings(page) if page.get_by_role("link", name="Next »").count() > 0: @@ -465,6 +779,13 @@ class WebInstaller(Installer): page.get_by_role("button", name="Continue to Matomo »").click() _wait_dom_settled(page) _page_warnings(page) + + page.wait_for_timeout(1_000) + if not is_installed(base_url): + _page_warnings(page) + raise RuntimeError( + "[install] Installer did not reach installed state." + ) except Exception as exc: _dump_failure_artifacts(page, reason=str(exc)) raise @@ -472,8 +793,4 @@ class WebInstaller(Installer): context.close() browser.close() - time.sleep(1) - if not is_installed(base_url): - raise RuntimeError("[install] Installer did not reach installed state.") - _log("[install] Installation finished.") diff --git a/tests/e2e/test_bootstrap_nix.py b/tests/e2e/test_bootstrap_nix.py index 1323c2c..0661fe4 100644 --- a/tests/e2e/test_bootstrap_nix.py +++ b/tests/e2e/test_bootstrap_nix.py @@ -1,5 +1,7 @@ import os +import re import subprocess +import textwrap import unittest @@ -7,14 +9,22 @@ MATOMO_URL = os.environ.get("MATOMO_URL", "http://127.0.0.1:8080") ADMIN_USER = os.environ.get("MATOMO_ADMIN_USER", "administrator") ADMIN_PASSWORD = os.environ.get("MATOMO_ADMIN_PASSWORD", "AdminSecret123!") ADMIN_EMAIL = os.environ.get("MATOMO_ADMIN_EMAIL", "administrator@example.org") +TOKEN_RE = re.compile(r"^[a-f0-9]{32,64}$") class TestMatomoBootstrapE2ENix(unittest.TestCase): def test_bootstrap_creates_api_token_via_nix(self) -> None: - script = f"""set -euo pipefail + script = textwrap.dedent( + f"""\ +set -eux export NIX_CONFIG='experimental-features = nix-command flakes' export TERM='xterm' +# Improve CI resilience for slow installer pages. +export MATOMO_INSTALLER_READY_TIMEOUT_S="${{MATOMO_INSTALLER_READY_TIMEOUT_S:-240}}" +export MATOMO_INSTALLER_STEP_TIMEOUT_S="${{MATOMO_INSTALLER_STEP_TIMEOUT_S:-45}}" +export MATOMO_INSTALLER_STEP_DEADLINE_S="${{MATOMO_INSTALLER_STEP_DEADLINE_S:-240}}" +export MATOMO_INSTALLER_DEBUG_DIR="${{MATOMO_INSTALLER_DEBUG_DIR:-/tmp/matomo-bootstrap}}" # Make sure we have a writable HOME (compose already sets HOME=/tmp/home) mkdir -p "$HOME" "$HOME/.cache" "$HOME/.config" "$HOME/.local/share" @@ -25,6 +35,16 @@ mkdir -p "$HOME" "$HOME/.cache" "$HOME/.config" "$HOME/.local/share" # Mark it as safe explicitly. git config --global --add safe.directory /work +# Preflight checks to surface "command not executable" failures (exit 126) clearly. +playwright_app="$(nix eval --raw .#apps.x86_64-linux.matomo-bootstrap-playwright-install.program)" +bootstrap_app="$(nix eval --raw .#apps.x86_64-linux.matomo-bootstrap.program)" +if [ -e "$playwright_app" ]; then + test -x "$playwright_app" +fi +if [ -e "$bootstrap_app" ]; then + test -x "$bootstrap_app" +fi + # 1) Install Playwright Chromium (cached in the container environment) nix run --no-write-lock-file -L .#matomo-bootstrap-playwright-install @@ -36,6 +56,7 @@ nix run --no-write-lock-file -L .#matomo-bootstrap -- \\ --admin-email '{ADMIN_EMAIL}' \\ --token-description 'e2e-test-token-nix' """ + ) cmd = [ "docker", @@ -50,8 +71,30 @@ nix run --no-write-lock-file -L .#matomo-bootstrap -- \\ script, ] - token = subprocess.check_output(cmd).decode().strip() - self.assertRegex(token, r"^[a-f0-9]{32,64}$") + result = subprocess.run( + cmd, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode != 0: + self.fail( + "nix bootstrap command failed\n" + f"exit={result.returncode}\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + + stdout_lines = [ + line.strip() for line in result.stdout.splitlines() if line.strip() + ] + token = stdout_lines[-1] if stdout_lines else "" + self.assertRegex( + token, + TOKEN_RE, + f"Expected token on last stdout line, got stdout={result.stdout!r}", + ) if __name__ == "__main__":