20 Commits

Author SHA1 Message Date
Kevin Veen-Birkenbach
3d5bfd5401 Release version 1.1.12
Some checks failed
ci / tests (push) Has been cancelled
ci / detect-release (push) Has been cancelled
ci / publish-image (push) Has been cancelled
ci / tag-stable (push) Has been cancelled
2026-02-14 19:05:51 +01:00
Kevin Veen-Birkenbach
fa6adea3c1 fix(installer): harden setupSuperUser race and add slow-resource e2e
Some checks failed
ci / tests (push) Has been cancelled
ci / detect-release (push) Has been cancelled
ci / publish-image (push) Has been cancelled
ci / tag-stable (push) Has been cancelled
2026-02-14 19:01:55 +01:00
Kevin Veen-Birkenbach
7ecb26cc92 Release version 1.1.11
Some checks failed
ci / tests (push) Has been cancelled
ci / detect-release (push) Has been cancelled
ci / publish-image (push) Has been cancelled
ci / tag-stable (push) Has been cancelled
2026-02-14 15:03:35 +01:00
Kevin Veen-Birkenbach
5d4a2d59db Fix installer selectors for setupSuperUser UI variants 2026-02-14 15:00:34 +01:00
Kevin Veen-Birkenbach
1847c14b63 Release version 1.1.10 2026-02-14 12:19:43 +01:00
Kevin Veen-Birkenbach
29e812f584 fix(installer): harden navigation races in setupSuperUser flow 2026-02-14 12:14:06 +01:00
Kevin Veen-Birkenbach
1c8de40a05 Release version 1.1.9 2026-02-14 11:04:30 +01:00
Kevin Veen-Birkenbach
4bfa7433f4 ci: run on all branches but release only from main via git tag detection
- Trigger CI on push for all branches and on pull_request
- Detect SemVer release tags (vX.Y.Z) via git tag --points-at
- Run publish-image and stable-tag only for tagged commits on main
- Pass version_tag and sha to reusable workflows
- Prevent tag pushes from triggering additional workflows

https://chatgpt.com/share/e/699044d3-c1d8-8013-a40d-974d1fc69974
2026-02-14 11:02:58 +01:00
Kevin Veen-Birkenbach
00c012e553 Release version 1.1.8 2026-02-14 10:53:40 +01:00
Kevin Veen-Birkenbach
1bebeb8abc ci: make ci.yml the single coordinator workflow
- Trigger ci on push and pull_request only
- Convert publish-image and stable-tag to reusable workflows (workflow_call)
- Add detect-release job for strict SemVer tag detection (vX.Y.Z)
- Run tests first, then publish image, then move stable tag
- Remove direct tag/push triggers from publish-image and stable-tag

https://chatgpt.com/share/e/699044d3-c1d8-8013-a40d-974d1fc69974
2026-02-14 10:47:56 +01:00
Kevin Veen-Birkenbach
01d1626cf2 ci(docker): publish image without leading 'v' in version tag
Strip the leading 'v' from git tags (e.g. v1.1.7 -> 1.1.7)
when pushing Docker images to GHCR.

https://chatgpt.com/share/e/699041b9-8610-8013-8d04-fbef93b10d3c
2026-02-14 10:34:41 +01:00
Kevin Veen-Birkenbach
5bbe78b272 Release version 1.1.7 2026-02-14 05:43:32 +01:00
Kevin Veen-Birkenbach
865d5155d5 Lint 2026-02-14 05:39:01 +01:00
Kevin Veen-Birkenbach
209037cd64 Harden compose installer timeouts and e2e stack diagnostics 2026-02-14 05:37:29 +01:00
Kevin Veen-Birkenbach
ba2d84b6cb Fix nix e2e to use compose run instead of exec 2026-02-14 05:08:12 +01:00
Kevin Veen-Birkenbach
4f5c41753f Release version 1.1.6 2026-02-14 04:54:10 +01:00
Kevin Veen-Birkenbach
aac01810a1 Add tables-step timeout env knobs to compose, docs, and e2e 2026-02-14 04:52:26 +01:00
Kevin Veen-Birkenbach
fb42167b89 Release version 1.1.5 2026-02-14 04:49:49 +01:00
Kevin Veen-Birkenbach
7836dbacf9 Harden web installer flow for nix e2e 2026-02-14 04:45:00 +01:00
Kevin Veen-Birkenbach
d380b1493c Lint 2026-02-14 03:47:37 +01:00
14 changed files with 1209 additions and 135 deletions

View File

@@ -1,11 +1,76 @@
name: ci name: ci
on: on:
pull_request: pull_request: {}
push: push:
branches: branches:
- main - "**"
permissions:
contents: write
packages: write
jobs: jobs:
tests: tests:
uses: ./.github/workflows/reusable-test.yml uses: ./.github/workflows/reusable-test.yml
with:
python-version: "3.12"
matomo-token-description: "ci-token"
detect-release:
# Only consider releases on main branch pushes (not PRs, not other branches)
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
outputs:
is_semver_tag: ${{ steps.detect.outputs.is_semver_tag }}
version_tag: ${{ steps.detect.outputs.version_tag }}
steps:
- name: Checkout (full history for tags)
uses: actions/checkout@v4
with:
fetch-depth: 0
- id: detect
shell: bash
run: |
set -euo pipefail
git fetch --tags --force
# Tags that point to the current commit
TAGS="$(git tag --points-at "$GITHUB_SHA" || true)"
# Pick the first strict SemVer tag: vX.Y.Z
VERSION_TAG="$(echo "$TAGS" | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -n1 || true)"
if [ -n "$VERSION_TAG" ]; then
echo "is_semver_tag=true" >> "$GITHUB_OUTPUT"
echo "version_tag=$VERSION_TAG" >> "$GITHUB_OUTPUT"
echo "Release tag detected on this commit: $VERSION_TAG"
else
echo "is_semver_tag=false" >> "$GITHUB_OUTPUT"
echo "version_tag=" >> "$GITHUB_OUTPUT"
echo "No SemVer tag on this commit."
fi
publish-image:
# Only on main, and only if detect-release found a SemVer tag on this commit
if: needs.detect-release.outputs.is_semver_tag == 'true'
needs: [tests, detect-release]
uses: ./.github/workflows/publish-image.yml
with:
version_tag: ${{ needs.detect-release.outputs.version_tag }}
sha: ${{ github.sha }}
permissions:
contents: read
packages: write
tag-stable:
# Only after tests + publish succeeded
if: needs.detect-release.outputs.is_semver_tag == 'true'
needs: [tests, detect-release, publish-image]
uses: ./.github/workflows/stable-tag.yml
with:
version_tag: ${{ needs.detect-release.outputs.version_tag }}
sha: ${{ github.sha }}
permissions:
contents: write

View File

@@ -1,32 +1,28 @@
name: publish-image name: publish-image
on: on:
push: workflow_call:
tags: inputs:
- "v*.*.*" version_tag:
type: string
workflow_run: required: true
workflows: ["Stable Tag"] # MUST match stable-tag.yml -> name: Stable Tag sha:
types: [completed] type: string
required: true
jobs: jobs:
build-and-push: build-and-push:
if: |
(github.event_name == 'push') ||
(github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')
runs-on: ubuntu-latest runs-on: ubuntu-latest
permissions: permissions:
contents: read contents: read
packages: write packages: write
steps: steps:
- name: Checkout - name: Checkout (exact commit)
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
# push: checks out the tag ref fetch-depth: 0
# workflow_run: checks out the exact commit that the Stable Tag workflow ran on ref: ${{ inputs.sha }}
ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.ref }}
- name: Set up QEMU - name: Set up QEMU
uses: docker/setup-qemu-action@v3 uses: docker/setup-qemu-action@v3
@@ -47,13 +43,9 @@ jobs:
run: | run: |
set -euo pipefail set -euo pipefail
IMAGE="ghcr.io/${{ github.repository }}" IMAGE="ghcr.io/${{ github.repository }}"
RAW_TAG="${{ inputs.version_tag }}" # e.g. v1.1.8
if [ "${{ github.event_name }}" = "push" ]; then TAG="${RAW_TAG#v}" # -> 1.1.8
TAG="${{ github.ref_name }}" # e.g. v1.1.0
echo "tags=$IMAGE:$TAG,$IMAGE:latest" >> "$GITHUB_OUTPUT" echo "tags=$IMAGE:$TAG,$IMAGE:latest" >> "$GITHUB_OUTPUT"
else
echo "tags=$IMAGE:stable" >> "$GITHUB_OUTPUT"
fi
- name: Build and push - name: Build and push
uses: docker/build-push-action@v6 uses: docker/build-push-action@v6

View File

@@ -1,24 +1,21 @@
name: Stable Tag name: Stable Tag
on: on:
push: workflow_call:
tags: inputs:
- "v*" version_tag:
type: string
required: true
sha:
type: string
required: true
permissions: permissions:
contents: write contents: write
jobs: jobs:
test:
uses: ./.github/workflows/reusable-test.yml
with:
python-version: "3.12"
matomo-token-description: "stable-ci-token"
tag-stable: tag-stable:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: test
if: startsWith(github.ref, 'refs/tags/v')
steps: steps:
- name: Checkout (full history for tags) - name: Checkout (full history for tags)
@@ -26,17 +23,17 @@ jobs:
with: with:
fetch-depth: 0 fetch-depth: 0
- name: Move stable tag to this version tag commit - name: Move stable tag to the release commit
shell: bash
run: | run: |
set -euo pipefail set -euo pipefail
echo "Triggered by tag: ${GITHUB_REF_NAME}" echo "Release tag: ${{ inputs.version_tag }}"
echo "Commit: ${GITHUB_SHA}" echo "Commit: ${{ inputs.sha }}"
git config user.name "github-actions[bot]" git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com" git config user.email "github-actions[bot]@users.noreply.github.com"
git fetch --tags --force git fetch --tags --force
git tag -fa stable -m "stable -> ${{ inputs.version_tag }} (${{ inputs.sha }})" "${{ inputs.sha }}"
git tag -fa stable -m "stable -> ${GITHUB_REF_NAME} (${GITHUB_SHA})" "${GITHUB_SHA}"
git push --force origin stable git push --force origin stable

View File

@@ -1,3 +1,43 @@
## [1.1.12] - 2026-02-14
* This release fixes the intermittent Matomo installer failure in the setupSuperUser step by adding more robust waiting logic and introduces E2E tests for deployments under very tight resource constraints.
## [1.1.11] - 2026-02-14
* This release improves matomo-bootstrap installer resilience by adding robust setupSuperUser field and button detection to prevent intermittent bootstrap failures.
## [1.1.10] - 2026-02-14
* This release fixes a reproducible Playwright navigation race in the Matomo installer (setupSuperUser), hardens the Next/Continue flow, and adds integration tests for transient locator errors and progress detection without a visible Next button.
## [1.1.9] - 2026-02-14
* Reworked CI to run on all branches while restricting Docker image publishing and stable tagging to tagged commits on main, using git-based SemVer detection.
## [1.1.8] - 2026-02-14
* Refactored CI to use a single coordinator workflow with strict SemVer-based release gating and adjusted Docker image publishing to strip the leading v from version tags.
## [1.1.7] - 2026-02-14
* Harden compose installer timeouts and e2e stack diagnostics
## [1.1.6] - 2026-02-14
* Add installer table-step timeout env vars (MATOMO_INSTALLER_TABLES_CREATION_TIMEOUT_S, MATOMO_INSTALLER_TABLES_ERASE_TIMEOUT_S) to compose/docs and e2e checks.
## [1.1.5] - 2026-02-14
* Harden web installer flow for nix e2e
## [1.1.4] - 2026-02-13 ## [1.1.4] - 2026-02-13
* This release hardens Matomo bootstrap by adding installer UI readiness waits/retries. * This release hardens Matomo bootstrap by adding installer UI readiness waits/retries.

View File

@@ -209,13 +209,15 @@ services:
MATOMO_TIMEZONE: "Germany - Berlin" MATOMO_TIMEZONE: "Germany - Berlin"
# Optional stability knobs # Optional stability knobs
MATOMO_TIMEOUT: "30" MATOMO_TIMEOUT: "60"
MATOMO_PLAYWRIGHT_HEADLESS: "1" MATOMO_PLAYWRIGHT_HEADLESS: "1"
MATOMO_PLAYWRIGHT_NAV_TIMEOUT_MS: "60000" MATOMO_PLAYWRIGHT_NAV_TIMEOUT_MS: "60000"
MATOMO_PLAYWRIGHT_SLOWMO_MS: "0" MATOMO_PLAYWRIGHT_SLOWMO_MS: "0"
MATOMO_INSTALLER_READY_TIMEOUT_S: "180" MATOMO_INSTALLER_READY_TIMEOUT_S: "240"
MATOMO_INSTALLER_STEP_TIMEOUT_S: "30" MATOMO_INSTALLER_STEP_TIMEOUT_S: "45"
MATOMO_INSTALLER_STEP_DEADLINE_S: "180" MATOMO_INSTALLER_STEP_DEADLINE_S: "240"
MATOMO_INSTALLER_TABLES_CREATION_TIMEOUT_S: "240"
MATOMO_INSTALLER_TABLES_ERASE_TIMEOUT_S: "180"
MATOMO_INSTALLER_DEBUG_DIR: "/tmp/matomo-bootstrap" MATOMO_INSTALLER_DEBUG_DIR: "/tmp/matomo-bootstrap"
restart: "no" restart: "no"

View File

@@ -61,13 +61,15 @@ services:
MATOMO_TIMEZONE: "Germany - Berlin" MATOMO_TIMEZONE: "Germany - Berlin"
# Optional stability knobs # Optional stability knobs
MATOMO_TIMEOUT: "30" MATOMO_TIMEOUT: "60"
MATOMO_PLAYWRIGHT_HEADLESS: "1" MATOMO_PLAYWRIGHT_HEADLESS: "1"
MATOMO_PLAYWRIGHT_NAV_TIMEOUT_MS: "60000" MATOMO_PLAYWRIGHT_NAV_TIMEOUT_MS: "60000"
MATOMO_PLAYWRIGHT_SLOWMO_MS: "0" MATOMO_PLAYWRIGHT_SLOWMO_MS: "0"
MATOMO_INSTALLER_READY_TIMEOUT_S: "180" MATOMO_INSTALLER_READY_TIMEOUT_S: "240"
MATOMO_INSTALLER_STEP_TIMEOUT_S: "30" MATOMO_INSTALLER_STEP_TIMEOUT_S: "45"
MATOMO_INSTALLER_STEP_DEADLINE_S: "180" MATOMO_INSTALLER_STEP_DEADLINE_S: "240"
MATOMO_INSTALLER_TABLES_CREATION_TIMEOUT_S: "240"
MATOMO_INSTALLER_TABLES_ERASE_TIMEOUT_S: "180"
MATOMO_INSTALLER_DEBUG_DIR: "/tmp/matomo-bootstrap" MATOMO_INSTALLER_DEBUG_DIR: "/tmp/matomo-bootstrap"
# bootstrap is a one-shot command that prints the token and exits # bootstrap is a one-shot command that prints the token and exits
# if you want to re-run, do: docker compose run --rm bootstrap # if you want to re-run, do: docker compose run --rm bootstrap

View File

@@ -29,7 +29,9 @@ MATOMO_TIMEZONE=Germany - Berlin
# MATOMO_PLAYWRIGHT_SLOWMO_MS=0 # MATOMO_PLAYWRIGHT_SLOWMO_MS=0
# Installer readiness / step guards # Installer readiness / step guards
# MATOMO_INSTALLER_READY_TIMEOUT_S=180 # MATOMO_INSTALLER_READY_TIMEOUT_S=240
# MATOMO_INSTALLER_STEP_TIMEOUT_S=30 # MATOMO_INSTALLER_STEP_TIMEOUT_S=45
# MATOMO_INSTALLER_STEP_DEADLINE_S=180 # MATOMO_INSTALLER_STEP_DEADLINE_S=240
# MATOMO_INSTALLER_TABLES_CREATION_TIMEOUT_S=240
# MATOMO_INSTALLER_TABLES_ERASE_TIMEOUT_S=180
# MATOMO_INSTALLER_DEBUG_DIR=/tmp/matomo-bootstrap # MATOMO_INSTALLER_DEBUG_DIR=/tmp/matomo-bootstrap

View File

@@ -20,7 +20,7 @@
rec { rec {
matomo-bootstrap = python.pkgs.buildPythonApplication { matomo-bootstrap = python.pkgs.buildPythonApplication {
pname = "matomo-bootstrap"; pname = "matomo-bootstrap";
version = "1.1.4"; # keep in sync with pyproject.toml version = "1.1.12"; # keep in sync with pyproject.toml
pyproject = true; pyproject = true;
src = self; src = self;

View File

@@ -4,12 +4,12 @@ build-backend = "setuptools.build_meta"
[project] [project]
name = "matomo-bootstrap" name = "matomo-bootstrap"
version = "1.1.4" version = "1.1.12"
description = "Headless bootstrap tooling for Matomo (installation + API token provisioning)" description = "Headless bootstrap tooling for Matomo (installation + API token provisioning)"
readme = "README.md" readme = "README.md"
requires-python = ">=3.10" requires-python = ">=3.10"
authors = [{ name = "Kevin Veen-Birkenbach", email = "kevin@veen.world" }] authors = [{ name = "Kevin Veen-Birkenbach", email = "kevin@veen.world" }]
license = { text = "MIT" } license = "MIT"
urls = { Homepage = "https://github.com/kevinveenbirkenbach/matomo-bootstrap" } urls = { Homepage = "https://github.com/kevinveenbirkenbach/matomo-bootstrap" }
dependencies = ["playwright>=1.46.0,<2"] dependencies = ["playwright>=1.46.0,<2"]

View File

@@ -28,6 +28,12 @@ INSTALLER_STEP_TIMEOUT_S = int(os.environ.get("MATOMO_INSTALLER_STEP_TIMEOUT_S",
INSTALLER_STEP_DEADLINE_S = int( INSTALLER_STEP_DEADLINE_S = int(
os.environ.get("MATOMO_INSTALLER_STEP_DEADLINE_S", "180") os.environ.get("MATOMO_INSTALLER_STEP_DEADLINE_S", "180")
) )
INSTALLER_TABLES_CREATION_TIMEOUT_S = int(
os.environ.get("MATOMO_INSTALLER_TABLES_CREATION_TIMEOUT_S", "180")
)
INSTALLER_TABLES_ERASE_TIMEOUT_S = int(
os.environ.get("MATOMO_INSTALLER_TABLES_ERASE_TIMEOUT_S", "120")
)
INSTALLER_DEBUG_DIR = os.environ.get( INSTALLER_DEBUG_DIR = os.environ.get(
"MATOMO_INSTALLER_DEBUG_DIR", "/tmp/matomo-bootstrap" "MATOMO_INSTALLER_DEBUG_DIR", "/tmp/matomo-bootstrap"
).rstrip("/") ).rstrip("/")
@@ -55,12 +61,89 @@ NEXT_BUTTON_CANDIDATES: list[tuple[str, str]] = [
("button", "Fortfahren"), ("button", "Fortfahren"),
] ]
CONTINUE_TO_MATOMO_CANDIDATES: list[tuple[str, str]] = [
("button", "Continue to Matomo »"),
("button", "Continue to Matomo"),
("link", "Continue to Matomo »"),
("link", "Continue to Matomo"),
]
SUPERUSER_LOGIN_SELECTORS = (
"#login-0",
"#login",
"input[name='login']",
"form#generalsetupform input[name='login']",
)
SUPERUSER_PASSWORD_SELECTORS = (
"#password-0",
"#password",
"input[name='password']",
"form#generalsetupform input[name='password']",
)
SUPERUSER_PASSWORD_REPEAT_SELECTORS = (
"#password_bis-0",
"#password_bis",
"input[name='password_bis']",
"form#generalsetupform input[name='password_bis']",
)
SUPERUSER_EMAIL_SELECTORS = (
"#email-0",
"#email",
"input[name='email']",
"form#generalsetupform input[name='email']",
)
SUPERUSER_SUBMIT_SELECTORS = (
"#submit-0",
"#submit",
"form#generalsetupform button[type='submit']",
"form#generalsetupform input[type='submit']",
)
FIRST_WEBSITE_NAME_SELECTORS = (
"#siteName-0",
"#siteName",
"input[name='siteName']",
"form#websitesetupform input[name='siteName']",
)
FIRST_WEBSITE_URL_SELECTORS = (
"#url-0",
"#url",
"input[name='url']",
"form#websitesetupform input[name='url']",
)
def _log(msg: str) -> None: def _log(msg: str) -> None:
# IMPORTANT: logs must not pollute stdout (tests expect only token on stdout) # IMPORTANT: logs must not pollute stdout (tests expect only token on stdout)
print(msg, file=sys.stderr) print(msg, file=sys.stderr)
_TRANSIENT_NAVIGATION_ERROR_SNIPPETS = (
"Execution context was destroyed",
"most likely because of a navigation",
"Cannot find context with specified id",
"Frame was detached",
)
def _is_transient_navigation_error(exc: Exception) -> bool:
msg = str(exc)
return any(snippet in msg for snippet in _TRANSIENT_NAVIGATION_ERROR_SNIPPETS)
def _count_locator(
locator, *, timeout_s: float = 2.0, retry_interval_s: float = 0.1
) -> int:
deadline = time.time() + timeout_s
while True:
try:
return locator.count()
except Exception as exc:
if _is_transient_navigation_error(exc) and time.time() < deadline:
time.sleep(retry_interval_s)
continue
raise
def _page_warnings(page, *, prefix: str = "[install]") -> list[str]: def _page_warnings(page, *, prefix: str = "[install]") -> list[str]:
""" """
Detect Matomo installer warnings/errors on the current page. Detect Matomo installer warnings/errors on the current page.
@@ -238,14 +321,14 @@ def _first_next_locator(page):
for role, name in NEXT_BUTTON_CANDIDATES: for role, name in NEXT_BUTTON_CANDIDATES:
loc = page.get_by_role(role, name=name) loc = page.get_by_role(role, name=name)
try: try:
if loc.count() > 0 and loc.first.is_visible(): if _count_locator(loc) > 0 and loc.first.is_visible():
return loc.first, f"{role}:{name}" return loc.first, f"{role}:{name}"
except Exception: except Exception:
continue continue
text_loc = page.get_by_text("Next", exact=False) text_loc = page.get_by_text("Next", exact=False)
try: try:
if text_loc.count() > 0 and text_loc.first.is_visible(): if _count_locator(text_loc) > 0 and text_loc.first.is_visible():
return text_loc.first, "text:Next*" return text_loc.first, "text:Next*"
except Exception: except Exception:
pass pass
@@ -253,11 +336,117 @@ def _first_next_locator(page):
return None, "" return None, ""
def _first_present_css_locator(page, selectors, *, timeout_s: float = 0.2):
for selector in selectors:
loc = page.locator(selector)
try:
if _count_locator(loc, timeout_s=timeout_s) > 0:
return loc.first, f"css:{selector}"
except Exception:
continue
return None, ""
def _first_continue_to_matomo_locator(page, *, timeout_s: float = 0.2):
for role, name in CONTINUE_TO_MATOMO_CANDIDATES:
loc = page.get_by_role(role, name=name)
try:
if _count_locator(loc, timeout_s=timeout_s) > 0 and loc.first.is_visible():
return loc.first, f"{role}:{name}"
except Exception:
continue
text_loc = page.get_by_text("Continue to Matomo", exact=False)
try:
if (
_count_locator(text_loc, timeout_s=timeout_s) > 0
and text_loc.first.is_visible()
):
return text_loc.first, "text:Continue to Matomo*"
except Exception:
pass
return None, ""
def _has_superuser_login_field(page, *, timeout_s: float = 0.2) -> bool:
loc, _ = _first_present_css_locator(
page, SUPERUSER_LOGIN_SELECTORS, timeout_s=timeout_s
)
return loc is not None
def _has_first_website_name_field(page, *, timeout_s: float = 0.2) -> bool:
loc, _ = _first_present_css_locator(
page, FIRST_WEBSITE_NAME_SELECTORS, timeout_s=timeout_s
)
return loc is not None
def _has_continue_to_matomo_action(page, *, timeout_s: float = 0.2) -> bool:
loc, _ = _first_continue_to_matomo_locator(page, timeout_s=timeout_s)
return loc is not None
def _wait_for_superuser_login_field(
page, *, timeout_s: float, poll_interval_ms: int = 300
) -> bool:
if timeout_s <= 0:
return _has_superuser_login_field(page, timeout_s=0.2)
deadline = time.time() + timeout_s
last_wait_log_at = 0.0
while time.time() < deadline:
_wait_dom_settled(page)
if _has_superuser_login_field(page, timeout_s=0.2):
return True
now = time.time()
if now - last_wait_log_at >= 5:
_log(
"[install] setupSuperUser reached but login form is not visible yet; "
f"waiting (url={page.url}, step={_get_step_hint(page.url)})"
)
_page_warnings(page)
last_wait_log_at = now
page.wait_for_timeout(poll_interval_ms)
return _has_superuser_login_field(page, timeout_s=0.2)
def _fill_required_input(page, selectors, value: str, *, label: str) -> None:
loc, _ = _first_present_css_locator(page, selectors, timeout_s=1.0)
if loc is None:
raise RuntimeError(
f"Could not locate required installer field '{label}' "
f"(url={page.url}, step={_get_step_hint(page.url)})."
)
try:
loc.click(timeout=2_000)
except Exception:
pass
loc.fill(value)
def _fill_optional_input(page, selectors, value: str) -> bool:
loc, _ = _first_present_css_locator(page, selectors, timeout_s=0.5)
if loc is None:
return False
try:
loc.click(timeout=2_000)
except Exception:
pass
loc.fill(value)
return True
def _installer_interactive(page) -> bool: def _installer_interactive(page) -> bool:
checks = [ checks = [
page.locator("#login-0").count() > 0, _has_superuser_login_field(page),
page.locator("#siteName-0").count() > 0, _has_first_website_name_field(page),
page.get_by_role("button", name="Continue to Matomo »").count() > 0, _has_continue_to_matomo_action(page),
] ]
loc, _ = _first_next_locator(page) loc, _ = _first_next_locator(page)
return any(checks) or loc is not None return any(checks) or loc is not None
@@ -280,12 +469,13 @@ def _wait_for_installer_interactive(page, *, timeout_s: int) -> None:
def _click_next_with_wait(page, *, timeout_s: int) -> str: def _click_next_with_wait(page, *, timeout_s: int) -> str:
before_url = page.url
before_step = _get_step_hint(before_url)
last_warning_log_at = 0.0
deadline = time.time() + timeout_s deadline = time.time() + timeout_s
while time.time() < deadline: while time.time() < deadline:
loc, label = _first_next_locator(page) loc, label = _first_next_locator(page)
if loc is not None: if loc is not None:
before_url = page.url
before_step = _get_step_hint(before_url)
try: try:
loc.click(timeout=2_000) loc.click(timeout=2_000)
except Exception: except Exception:
@@ -299,6 +489,44 @@ def _click_next_with_wait(page, *, timeout_s: int) -> str:
f"(url {before_url} -> {after_url})" f"(url {before_url} -> {after_url})"
) )
return after_step return after_step
_wait_dom_settled(page)
current_url = page.url
current_step = _get_step_hint(current_url)
if current_url != before_url or current_step != before_step:
_log(
"[install] Installer progressed without explicit click; "
f"step {before_step} -> {current_step} "
f"(url {before_url} -> {current_url})"
)
return current_step
# Some installer transitions render the next form asynchronously without
# exposing another "Next" control yet. Treat this as progress.
if _has_superuser_login_field(page, timeout_s=0.2):
_log(
"[install] Superuser form became available without explicit click; "
f"staying on step {current_step} (url {current_url})"
)
return current_step
if _has_first_website_name_field(page, timeout_s=0.2):
_log(
"[install] First website form became available without explicit click; "
f"staying on step {current_step} (url {current_url})"
)
return current_step
if _has_continue_to_matomo_action(page, timeout_s=0.2):
_log(
"[install] Continue-to-Matomo action is available without explicit click; "
f"staying on step {current_step} (url {current_url})"
)
return current_step
now = time.time()
if now - last_warning_log_at >= 5:
_page_warnings(page)
last_warning_log_at = now
page.wait_for_timeout(300) page.wait_for_timeout(300)
raise RuntimeError( raise RuntimeError(
@@ -307,6 +535,139 @@ def _click_next_with_wait(page, *, timeout_s: int) -> str:
) )
def _first_erase_tables_locator(page):
css_loc = page.locator("#eraseAllTables")
try:
if _count_locator(css_loc) > 0:
return css_loc.first, "css:#eraseAllTables"
except Exception:
pass
for role, name in [
("link", "Delete the detected tables »"),
("button", "Delete the detected tables »"),
("link", "Delete the detected tables"),
("button", "Delete the detected tables"),
]:
loc = page.get_by_role(role, name=name)
try:
if _count_locator(loc) > 0:
return loc.first, f"{role}:{name}"
except Exception:
continue
text_loc = page.get_by_text("Delete the detected tables", exact=False)
try:
if _count_locator(text_loc) > 0:
return text_loc.first, "text:Delete the detected tables*"
except Exception:
pass
return None, ""
def _resolve_tables_creation_conflict(page, *, timeout_s: int) -> bool:
before_url = page.url
before_step = _get_step_hint(before_url)
if "tablesCreation" not in before_step:
return False
loc, label = _first_erase_tables_locator(page)
if loc is None:
return False
_log(
"[install] Detected existing tables during tablesCreation. "
f"Trying cleanup via {label}."
)
def _cleanup_url() -> str | None:
try:
href = page.locator("#eraseAllTables").first.get_attribute("href")
if href:
return urllib.parse.urljoin(page.url, href)
except Exception:
pass
try:
parsed = urllib.parse.urlparse(page.url)
qs = urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
if (qs.get("action") or [""])[0] != "tablesCreation":
return None
qs["deleteTables"] = ["1"]
return urllib.parse.urlunparse(
parsed._replace(query=urllib.parse.urlencode(qs, doseq=True))
)
except Exception:
return None
deadline = time.time() + timeout_s
while time.time() < deadline:
accepted_dialog = False
def _accept_dialog(dialog) -> None:
nonlocal accepted_dialog
accepted_dialog = True
try:
_log(f"[install] Accepting installer dialog: {dialog.message}")
except Exception:
_log("[install] Accepting installer dialog.")
try:
dialog.accept()
except Exception:
pass
page.on("dialog", _accept_dialog)
try:
loc.click(timeout=2_000, force=True)
_wait_dom_settled(page)
except Exception as exc:
_log(f"[install] Cleanup click via {label} failed: {exc}")
cleanup_url = _cleanup_url()
if cleanup_url:
try:
page.goto(cleanup_url, wait_until="domcontentloaded")
_wait_dom_settled(page)
_log(
"[install] Triggered existing-table cleanup via URL fallback: "
f"{cleanup_url}"
)
except Exception as nav_exc:
_log(
"[install] Cleanup URL fallback failed: "
f"{cleanup_url} ({nav_exc})"
)
finally:
page.remove_listener("dialog", _accept_dialog)
if accepted_dialog:
_log("[install] Existing-table cleanup dialog accepted.")
_wait_dom_settled(page)
current_url = page.url
current_step = _get_step_hint(current_url)
if current_url != before_url or current_step != before_step:
_log(
"[install] Existing-table cleanup progressed installer; "
f"step {before_step} -> {current_step} "
f"(url {before_url} -> {current_url})"
)
return True
remaining_loc, _ = _first_erase_tables_locator(page)
if remaining_loc is None:
_log("[install] Existing-table cleanup control is gone.")
return True
loc = remaining_loc
page.wait_for_timeout(500)
raise RuntimeError(
"Detected existing Matomo tables but cleanup did not complete "
f"within {timeout_s}s (url={page.url}, step={_get_step_hint(page.url)})."
)
def wait_http(url: str, timeout: int = 180) -> None: def wait_http(url: str, timeout: int = 180) -> None:
""" """
Consider Matomo 'reachable' as soon as the HTTP server answers - even with 500. Consider Matomo 'reachable' as soon as the HTTP server answers - even with 500.
@@ -395,74 +756,265 @@ class WebInstaller(Installer):
try: try:
page.goto(base_url, wait_until="domcontentloaded") page.goto(base_url, wait_until="domcontentloaded")
_wait_for_installer_interactive(page, timeout_s=INSTALLER_READY_TIMEOUT_S) _wait_for_installer_interactive(
page, timeout_s=INSTALLER_READY_TIMEOUT_S
)
_page_warnings(page) _page_warnings(page)
progress_deadline = time.time() + INSTALLER_STEP_DEADLINE_S progress_deadline = time.time() + INSTALLER_STEP_DEADLINE_S
while page.locator("#login-0").count() == 0: while not _has_superuser_login_field(page):
if time.time() >= progress_deadline: now = time.time()
if now >= progress_deadline:
raise RuntimeError( raise RuntimeError(
"Installer did not reach superuser step " "Installer did not reach superuser step "
f"within {INSTALLER_STEP_DEADLINE_S}s " f"within {INSTALLER_STEP_DEADLINE_S}s "
f"(url={page.url}, step={_get_step_hint(page.url)})." f"(url={page.url}, step={_get_step_hint(page.url)})."
) )
current_step = _get_step_hint(page.url)
if "setupSuperUser" in current_step:
remaining_s = max(0.0, progress_deadline - now)
if _wait_for_superuser_login_field(page, timeout_s=remaining_s):
break
continue
if _resolve_tables_creation_conflict(
page, timeout_s=INSTALLER_TABLES_ERASE_TIMEOUT_S
):
_page_warnings(page)
continue
step_timeout = INSTALLER_STEP_TIMEOUT_S
if "tablesCreation" in _get_step_hint(page.url):
step_timeout = max(
step_timeout, INSTALLER_TABLES_CREATION_TIMEOUT_S
)
_click_next_with_wait(page, timeout_s=step_timeout)
_page_warnings(page)
_fill_required_input(
page,
SUPERUSER_LOGIN_SELECTORS,
config.admin_user,
label="superuser login",
)
_fill_required_input(
page,
SUPERUSER_PASSWORD_SELECTORS,
config.admin_password,
label="superuser password",
)
_fill_optional_input(
page, SUPERUSER_PASSWORD_REPEAT_SELECTORS, config.admin_password
)
_fill_required_input(
page,
SUPERUSER_EMAIL_SELECTORS,
config.admin_email,
label="superuser email",
)
_page_warnings(page)
submitted_superuser = False
try:
submitted_superuser = bool(
page.evaluate(
"""
([user, password, email]) => {
const form = document.querySelector("form#generalsetupform");
if (!form) return false;
const loginInput = form.querySelector("input[name='login']");
const passwordInput = form.querySelector("input[name='password']");
const repeatPasswordInput = form.querySelector("input[name='password_bis']");
const emailInput = form.querySelector("input[name='email']");
if (!loginInput || !passwordInput || !emailInput) return false;
loginInput.value = user;
passwordInput.value = password;
if (repeatPasswordInput) {
repeatPasswordInput.value = password;
}
emailInput.value = email;
if (typeof form.requestSubmit === "function") {
form.requestSubmit();
} else {
form.submit();
}
return true;
}
""",
[
config.admin_user,
config.admin_password,
config.admin_email,
],
)
)
except Exception:
submitted_superuser = False
if submitted_superuser:
_wait_dom_settled(page)
_log("[install] Submitted superuser form via form.requestSubmit().")
else:
submit_loc, submit_label = _first_present_css_locator(
page, SUPERUSER_SUBMIT_SELECTORS, timeout_s=0.5
)
if submit_loc is not None:
submit_loc.click(timeout=2_000)
_wait_dom_settled(page)
_log(
"[install] Submitted superuser form via "
f"{submit_label} fallback."
)
else:
_click_next_with_wait(page, timeout_s=INSTALLER_STEP_TIMEOUT_S) _click_next_with_wait(page, timeout_s=INSTALLER_STEP_TIMEOUT_S)
superuser_progress_deadline = time.time() + INSTALLER_STEP_TIMEOUT_S
while time.time() < superuser_progress_deadline:
_wait_dom_settled(page)
if not _has_superuser_login_field(page):
break
page.wait_for_timeout(300)
if _has_superuser_login_field(page):
_page_warnings(page)
raise RuntimeError(
"Superuser form submit did not progress to first website setup "
f"within {INSTALLER_STEP_TIMEOUT_S}s "
f"(url={page.url}, step={_get_step_hint(page.url)})."
)
_page_warnings(page) _page_warnings(page)
page.locator("#login-0").click() submitted_first_website = False
page.locator("#login-0").fill(config.admin_user) try:
submitted_first_website = bool(
page.evaluate(
"""
([siteName, siteUrl, timezoneLabel, ecommerceLabel]) => {
const form = document.querySelector("form#websitesetupform");
if (!form) return false;
page.locator("#password-0").click() const siteNameInput = form.querySelector("input[name='siteName']");
page.locator("#password-0").fill(config.admin_password) const siteUrlInput = form.querySelector("input[name='url']");
if (!siteNameInput || !siteUrlInput) return false;
if page.locator("#password_bis-0").count() > 0: siteNameInput.value = siteName;
page.locator("#password_bis-0").click() siteUrlInput.value = siteUrl;
page.locator("#password_bis-0").fill(config.admin_password)
page.locator("#email-0").click() const timezoneSelect = form.querySelector("select[name='timezone']");
page.locator("#email-0").fill(config.admin_email) if (timezoneSelect) {
_page_warnings(page) const timezoneOption = Array.from(timezoneSelect.options).find(
(opt) => (opt.textContent || "").trim() === timezoneLabel
);
if (timezoneOption) {
timezoneSelect.value = timezoneOption.value;
}
}
_click_next_with_wait(page, timeout_s=INSTALLER_STEP_TIMEOUT_S) const ecommerceSelect = form.querySelector("select[name='ecommerce']");
_page_warnings(page) if (ecommerceSelect) {
const ecommerceOption = Array.from(ecommerceSelect.options).find(
(opt) => (opt.textContent || "").trim() === ecommerceLabel
);
if (ecommerceOption) {
ecommerceSelect.value = ecommerceOption.value;
}
}
if page.locator("#siteName-0").count() > 0: if (typeof form.requestSubmit === "function") {
page.locator("#siteName-0").click() form.requestSubmit();
page.locator("#siteName-0").fill(DEFAULT_SITE_NAME) } else {
form.submit();
}
return true;
}
""",
[
DEFAULT_SITE_NAME,
DEFAULT_SITE_URL,
DEFAULT_TIMEZONE,
DEFAULT_ECOMMERCE,
],
)
)
except Exception:
submitted_first_website = False
if page.locator("#url-0").count() > 0: if submitted_first_website:
page.locator("#url-0").click() _wait_dom_settled(page)
page.locator("#url-0").fill(DEFAULT_SITE_URL) _log(
"[install] Submitted first website form via form.requestSubmit()."
)
else:
_fill_optional_input(
page, FIRST_WEBSITE_NAME_SELECTORS, DEFAULT_SITE_NAME
)
_fill_optional_input(
page, FIRST_WEBSITE_URL_SELECTORS, DEFAULT_SITE_URL
)
_page_warnings(page) _page_warnings(page)
try: try:
page.get_by_role("combobox").first.click() comboboxes = page.get_by_role("combobox")
page.get_by_role("listbox").get_by_text(DEFAULT_TIMEZONE).click() if _count_locator(comboboxes) > 0:
comboboxes.first.click(timeout=2_000)
page.get_by_role("listbox").get_by_text(
DEFAULT_TIMEZONE
).click(timeout=2_000)
except Exception: except Exception:
_log("Timezone selection skipped (not found / changed UI).") _log("Timezone selection skipped (not found / changed UI).")
try: try:
page.get_by_role("combobox").nth(2).click() comboboxes = page.get_by_role("combobox")
page.get_by_role("listbox").get_by_text(DEFAULT_ECOMMERCE).click() if _count_locator(comboboxes) > 2:
comboboxes.nth(2).click(timeout=2_000)
page.get_by_role("listbox").get_by_text(
DEFAULT_ECOMMERCE
).click(timeout=2_000)
except Exception: except Exception:
_log("Ecommerce selection skipped (not found / changed UI).") _log("Ecommerce selection skipped (not found / changed UI).")
_page_warnings(page) _page_warnings(page)
_click_next_with_wait(page, timeout_s=INSTALLER_STEP_TIMEOUT_S) _click_next_with_wait(page, timeout_s=INSTALLER_STEP_TIMEOUT_S)
first_website_progress_deadline = time.time() + INSTALLER_STEP_TIMEOUT_S
while time.time() < first_website_progress_deadline:
_wait_dom_settled(page)
if not _has_first_website_name_field(page):
break
page.wait_for_timeout(300)
if _has_first_website_name_field(page):
_page_warnings(page)
raise RuntimeError(
"First website form submit did not progress to tracking code "
f"within {INSTALLER_STEP_TIMEOUT_S}s "
f"(url={page.url}, step={_get_step_hint(page.url)})."
)
_page_warnings(page) _page_warnings(page)
if page.get_by_role("link", name="Next »").count() > 0: if _count_locator(page.get_by_role("link", name="Next »")) > 0:
page.get_by_role("link", name="Next »").click() page.get_by_role("link", name="Next »").click()
_wait_dom_settled(page) _wait_dom_settled(page)
_page_warnings(page) _page_warnings(page)
if page.get_by_role("button", name="Continue to Matomo »").count() > 0: continue_loc, _ = _first_continue_to_matomo_locator(page)
page.get_by_role("button", name="Continue to Matomo »").click() if continue_loc is not None:
continue_loc.click()
_wait_dom_settled(page) _wait_dom_settled(page)
_page_warnings(page) _page_warnings(page)
page.wait_for_timeout(1_000)
if not is_installed(base_url):
_page_warnings(page)
raise RuntimeError(
"[install] Installer did not reach installed state."
)
except Exception as exc: except Exception as exc:
_dump_failure_artifacts(page, reason=str(exc)) _dump_failure_artifacts(page, reason=str(exc))
raise raise
@@ -470,8 +1022,4 @@ class WebInstaller(Installer):
context.close() context.close()
browser.close() browser.close()
time.sleep(1)
if not is_installed(base_url):
raise RuntimeError("[install] Installer did not reach installed state.")
_log("[install] Installation finished.") _log("[install] Installation finished.")

View File

@@ -0,0 +1,32 @@
services:
db:
cpus: 0.35
mem_reservation: 192m
mem_limit: 320m
healthcheck:
interval: 10s
timeout: 5s
retries: 90
matomo:
cpus: 0.35
mem_reservation: 192m
mem_limit: 384m
healthcheck:
interval: 15s
timeout: 8s
retries: 120
start_period: 120s
bootstrap:
cpus: 0.75
mem_reservation: 512m
mem_limit: 1g
environment:
MATOMO_TIMEOUT: "120"
MATOMO_PLAYWRIGHT_NAV_TIMEOUT_MS: "120000"
MATOMO_INSTALLER_READY_TIMEOUT_S: "420"
MATOMO_INSTALLER_STEP_TIMEOUT_S: "120"
MATOMO_INSTALLER_STEP_DEADLINE_S: "420"
MATOMO_INSTALLER_TABLES_CREATION_TIMEOUT_S: "360"
MATOMO_INSTALLER_TABLES_ERASE_TIMEOUT_S: "240"

View File

@@ -1,5 +1,7 @@
import os import os
import re
import subprocess import subprocess
import textwrap
import unittest import unittest
@@ -7,14 +9,24 @@ MATOMO_URL = os.environ.get("MATOMO_URL", "http://127.0.0.1:8080")
ADMIN_USER = os.environ.get("MATOMO_ADMIN_USER", "administrator") ADMIN_USER = os.environ.get("MATOMO_ADMIN_USER", "administrator")
ADMIN_PASSWORD = os.environ.get("MATOMO_ADMIN_PASSWORD", "AdminSecret123!") ADMIN_PASSWORD = os.environ.get("MATOMO_ADMIN_PASSWORD", "AdminSecret123!")
ADMIN_EMAIL = os.environ.get("MATOMO_ADMIN_EMAIL", "administrator@example.org") ADMIN_EMAIL = os.environ.get("MATOMO_ADMIN_EMAIL", "administrator@example.org")
TOKEN_RE = re.compile(r"^[a-f0-9]{32,64}$")
class TestMatomoBootstrapE2ENix(unittest.TestCase): class TestMatomoBootstrapE2ENix(unittest.TestCase):
def test_bootstrap_creates_api_token_via_nix(self) -> None: def test_bootstrap_creates_api_token_via_nix(self) -> None:
script = f"""set -euo pipefail script = textwrap.dedent(
f"""\
set -eux
export NIX_CONFIG='experimental-features = nix-command flakes' export NIX_CONFIG='experimental-features = nix-command flakes'
export TERM='xterm' export TERM='xterm'
# Improve CI resilience for slow installer pages.
export MATOMO_INSTALLER_READY_TIMEOUT_S="${{MATOMO_INSTALLER_READY_TIMEOUT_S:-240}}"
export MATOMO_INSTALLER_STEP_TIMEOUT_S="${{MATOMO_INSTALLER_STEP_TIMEOUT_S:-45}}"
export MATOMO_INSTALLER_STEP_DEADLINE_S="${{MATOMO_INSTALLER_STEP_DEADLINE_S:-240}}"
export MATOMO_INSTALLER_TABLES_CREATION_TIMEOUT_S="${{MATOMO_INSTALLER_TABLES_CREATION_TIMEOUT_S:-240}}"
export MATOMO_INSTALLER_TABLES_ERASE_TIMEOUT_S="${{MATOMO_INSTALLER_TABLES_ERASE_TIMEOUT_S:-180}}"
export MATOMO_INSTALLER_DEBUG_DIR="${{MATOMO_INSTALLER_DEBUG_DIR:-/tmp/matomo-bootstrap}}"
# Make sure we have a writable HOME (compose already sets HOME=/tmp/home) # Make sure we have a writable HOME (compose already sets HOME=/tmp/home)
mkdir -p "$HOME" "$HOME/.cache" "$HOME/.config" "$HOME/.local/share" mkdir -p "$HOME" "$HOME/.cache" "$HOME/.config" "$HOME/.local/share"
@@ -25,6 +37,16 @@ mkdir -p "$HOME" "$HOME/.cache" "$HOME/.config" "$HOME/.local/share"
# Mark it as safe explicitly. # Mark it as safe explicitly.
git config --global --add safe.directory /work git config --global --add safe.directory /work
# Preflight checks to surface "command not executable" failures (exit 126) clearly.
playwright_app="$(nix eval --raw .#apps.x86_64-linux.matomo-bootstrap-playwright-install.program)"
bootstrap_app="$(nix eval --raw .#apps.x86_64-linux.matomo-bootstrap.program)"
if [ -e "$playwright_app" ]; then
test -x "$playwright_app"
fi
if [ -e "$bootstrap_app" ]; then
test -x "$bootstrap_app"
fi
# 1) Install Playwright Chromium (cached in the container environment) # 1) Install Playwright Chromium (cached in the container environment)
nix run --no-write-lock-file -L .#matomo-bootstrap-playwright-install nix run --no-write-lock-file -L .#matomo-bootstrap-playwright-install
@@ -36,13 +58,18 @@ nix run --no-write-lock-file -L .#matomo-bootstrap -- \\
--admin-email '{ADMIN_EMAIL}' \\ --admin-email '{ADMIN_EMAIL}' \\
--token-description 'e2e-test-token-nix' --token-description 'e2e-test-token-nix'
""" """
)
cmd = [ cmd = [
"docker", "docker",
"compose", "compose",
"-f", "-f",
"tests/e2e/docker-compose.yml", "tests/e2e/docker-compose.yml",
"exec", # Use `run` instead of `exec` to avoid runtime-specific
# `/etc/group` lookup issues seen with nix image + compose exec.
"run",
"--rm",
"--no-deps",
"-T", "-T",
"nix", "nix",
"sh", "sh",
@@ -50,8 +77,30 @@ nix run --no-write-lock-file -L .#matomo-bootstrap -- \\
script, script,
] ]
token = subprocess.check_output(cmd).decode().strip() result = subprocess.run(
self.assertRegex(token, r"^[a-f0-9]{32,64}$") cmd,
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
if result.returncode != 0:
self.fail(
"nix bootstrap command failed\n"
f"exit={result.returncode}\n"
f"stdout:\n{result.stdout}\n"
f"stderr:\n{result.stderr}"
)
stdout_lines = [
line.strip() for line in result.stdout.splitlines() if line.strip()
]
token = stdout_lines[-1] if stdout_lines else ""
self.assertRegex(
token,
TOKEN_RE,
f"Expected token on last stdout line, got stdout={result.stdout!r}",
)
if __name__ == "__main__": if __name__ == "__main__":

View File

@@ -7,13 +7,21 @@ import urllib.request
COMPOSE_FILE = os.environ.get("MATOMO_STACK_COMPOSE_FILE", "docker-compose.yml") COMPOSE_FILE = os.environ.get("MATOMO_STACK_COMPOSE_FILE", "docker-compose.yml")
SLOW_COMPOSE_FILE = os.environ.get(
"MATOMO_STACK_SLOW_COMPOSE_FILE", "tests/e2e/docker-compose.slow.yml"
)
# Pick a non-default port to avoid collisions with other CI stacks that use 8080 # Pick a non-default port to avoid collisions with other CI stacks that use 8080
MATOMO_PORT = os.environ.get("MATOMO_PORT", "18080") MATOMO_PORT = os.environ.get("MATOMO_PORT", "18080")
MATOMO_HOST_URL = os.environ.get("MATOMO_STACK_URL", f"http://127.0.0.1:{MATOMO_PORT}") MATOMO_HOST_URL = os.environ.get("MATOMO_STACK_URL", f"http://127.0.0.1:{MATOMO_PORT}")
MATOMO_SLOW_PORT = os.environ.get("MATOMO_SLOW_PORT", "18081")
MATOMO_SLOW_HOST_URL = os.environ.get(
"MATOMO_SLOW_STACK_URL", f"http://127.0.0.1:{MATOMO_SLOW_PORT}"
)
# How long we wait for Matomo HTTP to respond at all (seconds) # How long we wait for Matomo HTTP to respond at all (seconds)
WAIT_TIMEOUT_SECONDS = int(os.environ.get("MATOMO_STACK_WAIT_TIMEOUT", "180")) WAIT_TIMEOUT_SECONDS = int(os.environ.get("MATOMO_STACK_WAIT_TIMEOUT", "180"))
SLOW_WAIT_TIMEOUT_SECONDS = int(os.environ.get("MATOMO_SLOW_STACK_WAIT_TIMEOUT", "420"))
def _run( def _run(
@@ -32,8 +40,13 @@ def _run(
) )
def _compose_cmd(*args: str) -> list[str]: def _compose_cmd(*args: str, compose_files: list[str] | None = None) -> list[str]:
return ["docker", "compose", "-f", COMPOSE_FILE, *args] files = compose_files or [COMPOSE_FILE]
cmd = ["docker", "compose"]
for compose_file in files:
cmd.extend(["-f", compose_file])
cmd.extend(args)
return cmd
def _wait_for_http_any_status(url: str, timeout_s: int) -> None: def _wait_for_http_any_status(url: str, timeout_s: int) -> None:
@@ -108,31 +121,89 @@ class TestRootDockerComposeStack(unittest.TestCase):
extra_env={"MATOMO_PORT": MATOMO_PORT}, extra_env={"MATOMO_PORT": MATOMO_PORT},
) )
def test_root_docker_compose_yml_stack_bootstraps_and_token_works(self) -> None: def _assert_stack_bootstraps_and_token_works(
# Build bootstrap image from Dockerfile (as defined in docker-compose.yml) self,
*,
compose_files: list[str],
matomo_port: str,
matomo_host_url: str,
wait_timeout_seconds: int,
bootstrap_retries: int = 2,
) -> None:
build = _run( build = _run(
_compose_cmd("build", "bootstrap"), _compose_cmd("build", "bootstrap", compose_files=compose_files),
check=True, check=False,
extra_env={"MATOMO_PORT": MATOMO_PORT}, extra_env={"MATOMO_PORT": matomo_port},
)
self.assertEqual(
build.returncode,
0,
f"compose build failed\nstdout:\n{build.stdout}\nstderr:\n{build.stderr}",
) )
self.assertEqual(build.returncode, 0, build.stderr)
# Start db + matomo (bootstrap is one-shot and started via "run")
up = _run( up = _run(
_compose_cmd("up", "-d", "db", "matomo"), _compose_cmd("up", "-d", "db", "matomo", compose_files=compose_files),
check=True, check=False,
extra_env={"MATOMO_PORT": MATOMO_PORT}, extra_env={"MATOMO_PORT": matomo_port},
)
self.assertEqual(
up.returncode,
0,
f"compose up failed\nstdout:\n{up.stdout}\nstderr:\n{up.stderr}",
) )
self.assertEqual(up.returncode, 0, up.stderr)
# Wait until Matomo answers on the published port _wait_for_http_any_status(matomo_host_url + "/", wait_timeout_seconds)
_wait_for_http_any_status(MATOMO_HOST_URL + "/", WAIT_TIMEOUT_SECONDS)
# Run bootstrap: it should print ONLY the token to stdout boot_attempts: list[subprocess.CompletedProcess] = []
for _ in range(bootstrap_retries):
boot = _run( boot = _run(
_compose_cmd("run", "--rm", "bootstrap"), _compose_cmd("run", "--rm", "bootstrap", compose_files=compose_files),
check=True, check=False,
extra_env={"MATOMO_PORT": MATOMO_PORT}, extra_env={"MATOMO_PORT": matomo_port},
)
boot_attempts.append(boot)
if boot.returncode == 0:
break
time.sleep(5)
if boot.returncode != 0:
matomo_logs = _run(
_compose_cmd(
"logs",
"--no-color",
"--tail=250",
"matomo",
compose_files=compose_files,
),
check=False,
extra_env={"MATOMO_PORT": matomo_port},
)
db_logs = _run(
_compose_cmd(
"logs",
"--no-color",
"--tail=200",
"db",
compose_files=compose_files,
),
check=False,
extra_env={"MATOMO_PORT": matomo_port},
)
attempts_dump = "\n\n".join(
[
(
f"[attempt {i}] rc={attempt.returncode}\n"
f"stdout:\n{attempt.stdout}\n"
f"stderr:\n{attempt.stderr}"
)
for i, attempt in enumerate(boot_attempts, 1)
]
)
self.fail(
"bootstrap container failed after retry.\n"
f"{attempts_dump}\n\n"
f"[matomo logs]\n{matomo_logs.stdout}\n{matomo_logs.stderr}\n\n"
f"[db logs]\n{db_logs.stdout}\n{db_logs.stderr}"
) )
token = (boot.stdout or "").strip() token = (boot.stdout or "").strip()
@@ -142,9 +213,8 @@ class TestRootDockerComposeStack(unittest.TestCase):
f"Expected token_auth on stdout, got stdout={boot.stdout!r} stderr={boot.stderr!r}", f"Expected token_auth on stdout, got stdout={boot.stdout!r} stderr={boot.stderr!r}",
) )
# Verify token works against Matomo API
api_url = ( api_url = (
f"{MATOMO_HOST_URL}/index.php" f"{matomo_host_url}/index.php"
f"?module=API&method=SitesManager.getSitesWithAtLeastViewAccess" f"?module=API&method=SitesManager.getSitesWithAtLeastViewAccess"
f"&format=json&token_auth={token}" f"&format=json&token_auth={token}"
) )
@@ -153,6 +223,26 @@ class TestRootDockerComposeStack(unittest.TestCase):
self.assertIsInstance(data, list) self.assertIsInstance(data, list)
def test_root_docker_compose_yml_stack_bootstraps_and_token_works(self) -> None:
self._assert_stack_bootstraps_and_token_works(
compose_files=[COMPOSE_FILE],
matomo_port=MATOMO_PORT,
matomo_host_url=MATOMO_HOST_URL,
wait_timeout_seconds=WAIT_TIMEOUT_SECONDS,
bootstrap_retries=2,
)
def test_root_docker_compose_yml_stack_bootstraps_under_resource_pressure(
self,
) -> None:
self._assert_stack_bootstraps_and_token_works(
compose_files=[COMPOSE_FILE, SLOW_COMPOSE_FILE],
matomo_port=MATOMO_SLOW_PORT,
matomo_host_url=MATOMO_SLOW_HOST_URL,
wait_timeout_seconds=SLOW_WAIT_TIMEOUT_SECONDS,
bootstrap_retries=3,
)
class TestRootDockerComposeDefinition(unittest.TestCase): class TestRootDockerComposeDefinition(unittest.TestCase):
def test_bootstrap_service_waits_for_healthy_matomo_and_has_readiness_knobs( def test_bootstrap_service_waits_for_healthy_matomo_and_has_readiness_knobs(
@@ -173,11 +263,35 @@ class TestRootDockerComposeDefinition(unittest.TestCase):
self.assertIn("MATOMO_INSTALLER_READY_TIMEOUT_S:", bootstrap_block) self.assertIn("MATOMO_INSTALLER_READY_TIMEOUT_S:", bootstrap_block)
self.assertIn("MATOMO_INSTALLER_STEP_TIMEOUT_S:", bootstrap_block) self.assertIn("MATOMO_INSTALLER_STEP_TIMEOUT_S:", bootstrap_block)
self.assertIn("MATOMO_INSTALLER_STEP_DEADLINE_S:", bootstrap_block) self.assertIn("MATOMO_INSTALLER_STEP_DEADLINE_S:", bootstrap_block)
self.assertIn("MATOMO_INSTALLER_TABLES_CREATION_TIMEOUT_S:", bootstrap_block)
self.assertIn("MATOMO_INSTALLER_TABLES_ERASE_TIMEOUT_S:", bootstrap_block)
matomo_block = _extract_service_block(cfg.stdout, "matomo") matomo_block = _extract_service_block(cfg.stdout, "matomo")
self.assertIn("healthcheck:", matomo_block) self.assertIn("healthcheck:", matomo_block)
self.assertIn("curl -fsS http://127.0.0.1/ >/dev/null || exit 1", matomo_block) self.assertIn("curl -fsS http://127.0.0.1/ >/dev/null || exit 1", matomo_block)
def test_slow_override_sets_tight_resources_and_longer_timeouts(self) -> None:
cfg = _run(
_compose_cmd("config", compose_files=[COMPOSE_FILE, SLOW_COMPOSE_FILE]),
check=True,
extra_env={"MATOMO_PORT": MATOMO_SLOW_PORT},
)
self.assertEqual(cfg.returncode, 0, cfg.stderr)
matomo_block = _extract_service_block(cfg.stdout, "matomo")
self.assertIn("cpus: 0.35", matomo_block)
self.assertIn('mem_limit: "402653184"', matomo_block)
self.assertIn("start_period: 2m0s", matomo_block)
db_block = _extract_service_block(cfg.stdout, "db")
self.assertIn("cpus: 0.35", db_block)
self.assertIn('mem_limit: "335544320"', db_block)
bootstrap_block = _extract_service_block(cfg.stdout, "bootstrap")
self.assertIn("MATOMO_INSTALLER_STEP_TIMEOUT_S:", bootstrap_block)
self.assertIn("MATOMO_INSTALLER_STEP_DEADLINE_S:", bootstrap_block)
self.assertIn("MATOMO_INSTALLER_READY_TIMEOUT_S:", bootstrap_block)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()

View File

@@ -0,0 +1,231 @@
import unittest
from matomo_bootstrap.installers.web import (
_click_next_with_wait,
_count_locator,
_wait_for_superuser_login_field,
)
class _FlakyLocator:
def __init__(self, outcomes):
self._outcomes = list(outcomes)
self.calls = 0
def count(self) -> int:
self.calls += 1
outcome = self._outcomes.pop(0)
if isinstance(outcome, Exception):
raise outcome
return int(outcome)
class _StaticLocator:
def __init__(self, page, selector: str):
self._page = page
self._selector = selector
def count(self) -> int:
if self._selector == "#login-0":
return 1 if self._page.login_visible else 0
if self._selector == "#siteName-0":
return 0
return 0
@property
def first(self):
return self
def is_visible(self) -> bool:
return self.count() > 0
class _RoleLocator:
def __init__(self, count_value: int):
self._count_value = count_value
def count(self) -> int:
return self._count_value
@property
def first(self):
return self
def is_visible(self) -> bool:
return self._count_value > 0
class _NameOnlyStaticLocator:
def __init__(self, page, selector: str):
self._page = page
self._selector = selector
def count(self) -> int:
if self._selector == "input[name='login']":
return 1 if self._page.login_visible else 0
if self._selector == "input[name='siteName']":
return 0
return 0
@property
def first(self):
return self
def is_visible(self) -> bool:
return self.count() > 0
class _NoNextButLoginAppearsPage:
def __init__(self):
self.url = "http://matomo/index.php?action=setupSuperUser&module=Installation"
self.login_visible = False
self._wait_calls = 0
def locator(self, selector: str):
return _StaticLocator(self, selector)
def get_by_role(self, role: str, name: str):
return _RoleLocator(0)
def get_by_text(self, *_args, **_kwargs):
return _RoleLocator(0)
def title(self) -> str:
return "setupSuperUser"
def wait_for_load_state(self, *_args, **_kwargs):
return None
def wait_for_timeout(self, *_args, **_kwargs):
self._wait_calls += 1
if self._wait_calls >= 1:
self.login_visible = True
class _NoNextButNamedLoginAppearsPage:
def __init__(self):
self.url = "http://matomo/index.php?action=setupSuperUser&module=Installation"
self.login_visible = False
self._wait_calls = 0
def locator(self, selector: str):
return _NameOnlyStaticLocator(self, selector)
def get_by_role(self, role: str, name: str):
return _RoleLocator(0)
def get_by_text(self, *_args, **_kwargs):
return _RoleLocator(0)
def title(self) -> str:
return "setupSuperUser"
def wait_for_load_state(self, *_args, **_kwargs):
return None
def wait_for_timeout(self, *_args, **_kwargs):
self._wait_calls += 1
if self._wait_calls >= 1:
self.login_visible = True
class _DelayedSuperuserLoginPage:
def __init__(self, *, reveal_after_wait_calls: int | None):
self.url = "http://matomo/index.php?action=setupSuperUser&module=Installation"
self.login_visible = False
self._wait_calls = 0
self._reveal_after_wait_calls = reveal_after_wait_calls
def locator(self, selector: str):
return _StaticLocator(self, selector)
def get_by_role(self, role: str, name: str):
return _RoleLocator(0)
def get_by_text(self, *_args, **_kwargs):
return _RoleLocator(0)
def title(self) -> str:
return "setupSuperUser"
def wait_for_load_state(self, *_args, **_kwargs):
return None
def wait_for_timeout(self, *_args, **_kwargs):
self._wait_calls += 1
if (
self._reveal_after_wait_calls is not None
and self._wait_calls >= self._reveal_after_wait_calls
):
self.login_visible = True
class TestWebInstallerLocatorCountIntegration(unittest.TestCase):
def test_retries_transient_navigation_error(self) -> None:
locator = _FlakyLocator(
[
RuntimeError(
"Locator.count: Execution context was destroyed, most likely because of a navigation"
),
RuntimeError(
"Locator.count: Execution context was destroyed, most likely because of a navigation"
),
1,
]
)
result = _count_locator(locator, timeout_s=0.5, retry_interval_s=0.0)
self.assertEqual(result, 1)
self.assertEqual(locator.calls, 3)
def test_raises_non_transient_error_without_retry(self) -> None:
locator = _FlakyLocator([RuntimeError("Locator is not attached to DOM")])
with self.assertRaises(RuntimeError):
_count_locator(locator, timeout_s=0.5, retry_interval_s=0.0)
self.assertEqual(locator.calls, 1)
def test_click_next_wait_treats_login_form_as_progress(self) -> None:
page = _NoNextButLoginAppearsPage()
step = _click_next_with_wait(page, timeout_s=1)
self.assertEqual(step, "Installation:setupSuperUser")
self.assertTrue(page.login_visible)
def test_click_next_wait_treats_named_login_form_as_progress(self) -> None:
page = _NoNextButNamedLoginAppearsPage()
step = _click_next_with_wait(page, timeout_s=1)
self.assertEqual(step, "Installation:setupSuperUser")
self.assertTrue(page.login_visible)
def test_wait_for_superuser_login_field_allows_delayed_form(self) -> None:
page = _DelayedSuperuserLoginPage(reveal_after_wait_calls=4)
visible = _wait_for_superuser_login_field(
page,
timeout_s=1.0,
poll_interval_ms=1,
)
self.assertTrue(visible)
self.assertTrue(page.login_visible)
def test_wait_for_superuser_login_field_times_out_when_absent(self) -> None:
page = _DelayedSuperuserLoginPage(reveal_after_wait_calls=None)
visible = _wait_for_superuser_login_field(
page,
timeout_s=0.01,
poll_interval_ms=1,
)
self.assertFalse(visible)
if __name__ == "__main__":
unittest.main()