From 1e25fb947d3c01fc983b481fff5b064ed82349ab Mon Sep 17 00:00:00 2001 From: Christian Cleberg Date: Mon, 20 Jul 2026 17:50:56 -0500 Subject: ci: split accessibility coverage between local runtimes and CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two-job CI matrix was buying two near-identical iOS 26.x runs at double the macOS minutes. GitHub's macos-26 image ships only 26.x simulator runtimes, so it cannot test the 17.6 floor at all, and floor coverage was the entire justification for the second job. Split the work by what each side can uniquely do instead: CI keeps one job on the newest runtime. Its real value is not the runtime — it is building a clean checkout of the merge result, which catches a file that was never committed. A local run cannot, and that failure mode is live here: DomainDig.xcodeproj is hand-edited and uses file-system-synchronized groups, where an entire missing folder still builds locally. sr.ht cannot run macOS, so this is the only place that check exists. Collapsing the matrix also removed the deployment-target math, since "newest" is always above the floor. Scripts/audit-a11y.sh runs the audit against real runtimes, defaulting to floor + current. It reads the deployment target from the project rather than hard-coding it, selects the oldest runtime at or above it (one below is useless — the app cannot install), and says so plainly when the nearest installed runtime is a major version above the target rather than implying floor coverage it does not have. .githooks/pre-push runs the floor tier, and only when Swift, asset, or project files changed. Pre-push rather than pre-commit because the suite takes ~85s: at pre-commit that blocks every commit, and a hook routinely bypassed with --no-verify is worse than none. Opt in per clone with 'git config core.hooksPath .githooks'. Docs/ACCESSIBILITY.md records the split, the measured non-nested coverage that motivates it, and the enforcement ratchet. --- .githooks/pre-push | 68 +++++++++++++++++++++++ .github/workflows/build.yml | 92 +++++++++++-------------------- Docs/ACCESSIBILITY.md | 96 +++++++++++++++++++++++++++++++++ README.md | 24 +++++++++ Scripts/audit-a11y.sh | 129 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 348 insertions(+), 61 deletions(-) create mode 100755 .githooks/pre-push create mode 100644 Docs/ACCESSIBILITY.md create mode 100755 Scripts/audit-a11y.sh diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100755 index 0000000..aff4368 --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# +# Run the accessibility audit against the oldest supported simulator runtime +# before pushing. +# +# Enable once per clone: +# git config core.hooksPath .githooks +# +# Why pre-push and not pre-commit: the suite takes ~85s. At pre-commit that +# blocks every commit, and a hook you routinely bypass with --no-verify is worse +# than no hook, because it trains you to ignore it. Pushes are far less frequent +# and map to the unit of work that actually reaches CI. +# +# Why only the floor tier: CI already audits the newest runtime on a clean +# checkout. The GitHub image has no old runtimes, so the floor is the one thing +# CI structurally cannot cover — and it is the one this machine can. Running +# both here would just double the wait to re-check what CI already does. +# +# Skip deliberately with: git push --no-verify + +set -euo pipefail + +repo_root=$(git rev-parse --show-toplevel) +cd "$repo_root" + +zero='0000000000000000000000000000000000000000' +changed='' + +# stdin: , one line per ref. +while read -r _local_ref local_sha _remote_ref remote_sha; do + [ "$local_sha" = "$zero" ] && continue # branch deletion + + if [ "$remote_sha" = "$zero" ]; then + # New branch: diff against the default branch rather than the whole history. + base=$(git merge-base origin/main "$local_sha" 2>/dev/null || echo '') + range="${base:+$base..}$local_sha" + else + range="$remote_sha..$local_sha" + fi + + changed="$changed$(git diff --name-only "$range" 2>/dev/null || true)"$'\n' +done + +if [ -z "$(printf '%s' "$changed" | tr -d '[:space:]')" ]; then + exit 0 +fi + +# Only pay the ~85s when something could actually change the rendered UI. +if ! printf '%s' "$changed" | grep -qE '\.(swift|xcassets|xcodeproj)|\.pbxproj|xcscheme'; then + echo "pre-push: no Swift/project changes, skipping accessibility audit" + exit 0 +fi + +echo "pre-push: running accessibility audit on the floor runtime (~85s)" +echo " skip with 'git push --no-verify'" + +if ! ./Scripts/audit-a11y.sh floor; then + echo + echo "pre-push: audit could not run. Push aborted." >&2 + echo " Re-run with './Scripts/audit-a11y.sh floor' to see why," >&2 + echo " or bypass with 'git push --no-verify'." >&2 + exit 1 +fi + +# Note: findings are reported, not enforced, so a clean exit here does not mean +# zero findings — read the list above. Enforcement is controlled by +# AccessibilityAuditHarness.enforcedAuditTypes. +exit 0 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e0c2eee..5d9d877 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,39 +9,29 @@ name: Build # dependencies). The test target is DomainDigUITests — an accessibility audit # suite; see DomainDigUITests/AccessibilityAuditHarness.swift. # +# WHAT THIS JOB IS FOR, given the audit also runs locally: +# a clean checkout of the merge result. A local hook runs against the working +# tree and therefore cannot catch a file that was never committed — the failure +# mode that matters most here, since DomainDig.xcodeproj is hand-edited and uses +# file-system-synchronized groups where a whole missing folder still builds fine +# locally. This job is the only place that check exists; sr.ht cannot run it. +# +# DELIBERATELY ONE JOB, NEWEST RUNTIME ONLY. Audit coverage is not nested across +# OS versions, so the oldest supported OS genuinely needs its own run — but the +# macos-26 image ships only iOS 26.x runtimes, so CI *cannot* provide it. Asking +# for two jobs here bought two near-identical 26.x runs at double the macOS +# minutes. Floor coverage lives in Scripts/audit-a11y.sh, run from a machine that +# actually has an 18.x runtime installed, and is wired to the pre-push hook in +# .githooks/. See Docs/ACCESSIBILITY.md for the split. +# # The audit REPORTS but does not FAIL by default. It surfaces violations that # exist today, so gating on it would block every unrelated PR until the # accessibility pass in issue #21 completes. Findings land in the job log and in -# the uploaded .xcresult bundle, tagged [report] or [FAIL]. -# -# Enforcement is a committed constant, not a CI setting: widen -# `AccessibilityAuditHarness.enforcedAuditTypes` as each phase clears a category. -# (Env vars were tried first — neither a plain xcodebuild env var nor a -# TEST_RUNNER_-prefixed build setting reaches the UI test process.) -# -# The matrix runs two simulators because audit coverage is NOT nested — each -# runtime reports findings the other misses, in both directions. Measured -# locally on the Tracked Domains screen, iOS 18.6 reported 2 issues and iOS 27.0 -# reported 6 (including contrast and element-detection issues 18.6 never -# raised); at accessibility text sizes the Dashboard produced a hit-region -# finding on 18.6 that 27.0 did not. -# -# CAVEAT — this image cannot test the real support floor. The app's deployment -# target is 17.6, but the macos-26 runner ships only iOS 26.x simulator -# runtimes, so "floor" resolves to ~26.2 here rather than an 18.x image. CI -# therefore compares two 26.x runtimes; genuine oldest-supported-OS coverage has -# to come from a local run or a self-hosted runner with older runtimes -# installed. The "Select simulator" step emits a warning annotation when the -# resolved floor sits well above the deployment target, so this gap stays -# visible instead of being silently assumed away. -# -# Installing an older runtime in CI (xcodebuild -downloadPlatform iOS -# -buildVersion 18.6) is possible but costs several GB and minutes per job; not -# done by default. -# -# Runtimes are resolved dynamically rather than pinned. The previous selector -# took the first iPhone from any runtime, which could pick a simulator BELOW the -# deployment target, where the app cannot install. +# the uploaded .xcresult bundle, tagged [report] or [FAIL]. Enforcement is a +# committed constant: widen `AccessibilityAuditHarness.enforcedAuditTypes` as +# each phase clears a category. (Env vars were tried first — neither a plain +# xcodebuild env var nor a TEST_RUNNER_-prefixed build setting reaches the UI +# test process.) # # pull_request only, plus manual dispatch. GitHub builds the merge result (PR # merged into main), so a green PR validates exactly what will land on main. @@ -65,25 +55,13 @@ concurrency: group: build-${{ github.ref }} cancel-in-progress: true -env: - # Keep in sync with IPHONEOS_DEPLOYMENT_TARGET in DomainDig.xcodeproj. - DEPLOYMENT_TARGET_MAJOR: '17' - DEPLOYMENT_TARGET_MINOR: '6' - jobs: test: - name: xcodebuild test (${{ matrix.tier }}) + name: xcodebuild test # macos-latest still points at macOS 15, which lacks the iOS 26+ SDK this app # is built against. runs-on: macos-26 - strategy: - fail-fast: false - matrix: - # floor = oldest runtime the app actually supports - # current = newest runtime available on the image - tier: [floor, current] - steps: - uses: actions/checkout@v7 @@ -96,26 +74,27 @@ jobs: id: sim run: | set -euo pipefail - floor=$(( DEPLOYMENT_TARGET_MAJOR * 1000 + DEPLOYMENT_TARGET_MINOR )) + # Newest available iPhone runtime. No deployment-target filtering is + # needed for "newest" — it is always at or above the floor. The + # previous selector took the first iPhone from ANY runtime, which on a + # machine with an older runtime installed could pick a simulator below + # the deployment target, where the app cannot install. selected=$(xcrun simctl list devices available --json \ - | jq -c --argjson floor "$floor" --arg tier "${{ matrix.tier }}" ' + | jq -c ' [ .devices | to_entries[] | (.key | capture("SimRuntime\\.iOS-(?[0-9]+)-(?[0-9]+)$")) as $v | (($v.maj | tonumber) * 1000 + ($v.min | tonumber)) as $rank - | select($rank >= $floor) | .value[] | select(.name | startswith("iPhone")) | { rank: $rank, udid: .udid, name: .name, os: "\($v.maj).\($v.min)" } ] | sort_by(.rank, .name) - | if length == 0 then empty - elif $tier == "floor" then .[0] - else .[-1] end + | last ') - if [ -z "$selected" ]; then - echo "::error::No iPhone simulator at or above iOS ${DEPLOYMENT_TARGET_MAJOR}.${DEPLOYMENT_TARGET_MINOR} on this image" + if [ -z "$selected" ] || [ "$selected" = "null" ]; then + echo "::error::No iPhone simulator available on this image" xcrun simctl list devices available >&2 exit 1 fi @@ -125,15 +104,6 @@ jobs: echo "udid=$(echo "$selected" | jq -r .udid)" >> "$GITHUB_OUTPUT" echo "label=$label" >> "$GITHUB_OUTPUT" - # Surface the floor-coverage gap rather than letting the matrix imply - # coverage it does not have. One major version of slack is tolerated. - if [ "${{ matrix.tier }}" = "floor" ]; then - rank=$(echo "$selected" | jq -r .rank) - if [ "$rank" -ge $(( (DEPLOYMENT_TARGET_MAJOR + 1) * 1000 )) ]; then - echo "::warning::Floor tier resolved to $label, well above the ${DEPLOYMENT_TARGET_MAJOR}.${DEPLOYMENT_TARGET_MINOR} deployment target. This image has no older runtime, so the oldest supported OS is NOT covered by this run." - fi - fi - - name: Test on ${{ steps.sim.outputs.label }} run: | set -o pipefail @@ -150,6 +120,6 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: test-results-${{ matrix.tier }} + name: test-results path: TestResults.xcresult retention-days: 7 diff --git a/Docs/ACCESSIBILITY.md b/Docs/ACCESSIBILITY.md new file mode 100644 index 0000000..1045df9 --- /dev/null +++ b/Docs/ACCESSIBILITY.md @@ -0,0 +1,96 @@ +# Accessibility Audit + +`DomainDigUITests` runs Apple's `performAccessibilityAudit()` across every +primary screen. The audit checks contrast, hit-region size, clipped text at +large Dynamic Type, element descriptions, trait correctness, and Dynamic Type +support — the same ground the accessibility pass tracked in +[issue #21](https://github.com/zerolabsco/domain-dig/issues/21) covers. + +## Findings are reported, not enforced + +The audit surfaces violations that exist today, so failing on all of them would +block every unrelated change until the whole pass lands. Instead, findings are +logged and attached to the result bundle tagged `[report]` or `[FAIL]`. + +Enforcement is the committed constant +`AccessibilityAuditHarness.enforcedAuditTypes`. Widen it as each phase clears a +category: + +| After phase | Enforce | +| --- | --- | +| 2 — semantic colors + light mode | `.contrast` | +| 3 — Dynamic Type + reflow | `.textClipped`, `.dynamicType`, `.hitRegion` | +| 4 — VoiceOver | `.elementDetection`, `.sufficientElementDescription`, `.trait` | + +A constant rather than a CI setting, for two reasons. Environment variables do +not work: neither a plain `xcodebuild` env var nor a `TEST_RUNNER_`-prefixed +build setting reaches the UI test process, so the toggle silently did nothing. +And a committed value makes "when did contrast become enforced?" answerable with +`git blame` instead of CI tribal knowledge. + +## Why coverage is split between local and CI + +**Audit coverage is not nested across OS versions.** Each runtime reports +findings the others miss, in *both* directions. Measured on this project: + +| Screen | iOS 18.6 | iOS 27.0 | +| --- | --- | --- | +| Tracked Domains | 2 (text clipped) | **6** (+ contrast ×3, element detection) | +| Settings | 2 contrast | **`dynamicType`** finding 18.6 missed | +| Dashboard @ `AccessibilityXXXL` | **hit region** + 2 clipped | 1 clipped only | + +Neither runtime is a superset, so the oldest supported OS needs its own run. +This also rules out committing per-screen baseline counts as a regression guard: +no single number is correct on both. + +The catch is that **GitHub's `macos-26` image ships only iOS 26.x simulator +runtimes.** It cannot test the 17.6 floor at all. A two-job CI matrix was tried +and produced two near-identical 26.x runs at double the macOS minutes. + +So the work is split by what each side can uniquely do: + +| | Runtime | Uniquely provides | +| --- | --- | --- | +| **CI** (`.github/workflows/build.yml`) | newest available | A clean checkout of the merge result — catches a file that was never committed, which a local run cannot. Matters here because `DomainDig.xcodeproj` is hand-edited and uses file-system-synchronized groups, where a whole missing folder still builds locally. | +| **Local** (`Scripts/audit-a11y.sh`) | oldest supported + newest | Real floor coverage, on a machine that actually has an 18.x runtime installed. | + +Together they cover both ends; neither duplicates the other. + +## Running it + +```sh +./Scripts/audit-a11y.sh # floor + current +./Scripts/audit-a11y.sh floor # oldest supported only (~85s) +./Scripts/audit-a11y.sh current # newest installed only +``` + +The script reads the deployment target from the project rather than hard-coding +it, and selects the oldest installed runtime **at or above** it — a runtime +below the deployment target is useless, because the app cannot install there. +If the nearest installed runtime is a major version above the target, it says +so rather than implying floor coverage it does not have. + +### Pre-push hook + +```sh +git config core.hooksPath .githooks +``` + +Runs the floor audit before a push, and only when Swift, asset, or project files +changed. Bypass with `git push --no-verify`. + +Pre-push rather than pre-commit deliberately: the suite takes ~85s, and at +pre-commit that blocks every commit. A hook routinely bypassed with +`--no-verify` is worse than no hook, because it trains you to ignore it. + +## Notes + +- Audits retry up to three times. Slower machines can miss the audit's internal + deadline (`Audit failed to complete in time`, code `-56`), which is a tooling + timeout, not an app defect. A screen that still cannot be audited is reported + as an `XCTSkip`, never a pass — skips are visually distinct in CI, so an + unaudited screen stays visible instead of being silently counted as clean. +- The suite launches with `DOMAIN_DIG_FORCE_PRO_PLUS` so Pro-gated screens are + reachable. `PurchaseService` honours that argument in `DEBUG` builds only. +- Everything used is available at the iOS 17.6 deployment floor; + `performAccessibilityAudit` is `ios(17.0)`. diff --git a/README.md b/README.md index 736418d..6d89ceb 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,30 @@ xcodebuild -project DomainDig.xcodeproj -scheme DomainDig -destination 'platform The app and local API share the canonical report pipeline through `DomainInspectionService`, `DomainReportBuilder`, and `DomainReportExporter`. +### Accessibility Audit + +`DomainDigUITests` runs `performAccessibilityAudit()` over every primary screen, +at default and at the largest accessibility text size. + +```sh +./Scripts/audit-a11y.sh # oldest supported + newest runtime +./Scripts/audit-a11y.sh floor # oldest supported runtime only (~85s) +``` + +Findings are **reported, not enforced** — they are the burndown list for the +accessibility pass. Widen `AccessibilityAuditHarness.enforcedAuditTypes` to turn +a category into a build failure as each phase lands. + +Optional pre-push hook, which runs the floor audit when Swift or project files +change: + +```sh +git config core.hooksPath .githooks +``` + +See [Docs/ACCESSIBILITY.md](Docs/ACCESSIBILITY.md) for why coverage is split +between this script and CI. + ## Release Planning See `RELEASE_ROADMAP.md` for the semver release plan from `v4.4.1` through the planned `v5.0.0` stabilization milestone. diff --git a/Scripts/audit-a11y.sh b/Scripts/audit-a11y.sh new file mode 100755 index 0000000..9ac3aa0 --- /dev/null +++ b/Scripts/audit-a11y.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# +# Run the accessibility audit suite against real simulator runtimes. +# +# ./Scripts/audit-a11y.sh # floor + current +# ./Scripts/audit-a11y.sh floor # oldest supported runtime only +# ./Scripts/audit-a11y.sh current # newest installed runtime only +# +# Why this exists rather than living entirely in CI: audit coverage is NOT +# nested across OS versions — each runtime reports findings the others miss, in +# both directions. Measured on Tracked Domains, iOS 18.6 reported 2 findings and +# iOS 27.0 reported 6; at accessibility text sizes the Dashboard produced a +# hit-region finding on 18.6 that 27.0 did not. The GitHub macos-26 image ships +# only iOS 26.x runtimes, so it structurally cannot cover the floor. This machine +# can. +# +# CI covers "current" on a clean checkout (which a local run cannot, since it +# would miss an uncommitted file). This script covers "floor" (which CI cannot). +# Together they cover both; neither duplicates the other. +# +# The deployment target is read from the project rather than hard-coded, so it +# cannot drift out of sync. + +set -euo pipefail + +cd "$(dirname "$0")/.." + +PROJECT="DomainDig.xcodeproj" +SCHEME="DomainDig" +TIER="${1:-both}" + +case "$TIER" in + floor | current | both) ;; + *) + echo "usage: $0 [floor|current|both]" >&2 + exit 2 + ;; +esac + +echo "==> Reading deployment target from $PROJECT" +DEPLOYMENT_TARGET=$( + xcodebuild -project "$PROJECT" -scheme "$SCHEME" -showBuildSettings 2>/dev/null \ + | awk -F' = ' '/ IPHONEOS_DEPLOYMENT_TARGET = /{print $2; exit}' +) + +if [ -z "${DEPLOYMENT_TARGET:-}" ]; then + echo "error: could not read IPHONEOS_DEPLOYMENT_TARGET" >&2 + exit 1 +fi + +DT_MAJOR="${DEPLOYMENT_TARGET%%.*}" +DT_MINOR="${DEPLOYMENT_TARGET##*.}" +[ "$DT_MINOR" = "$DEPLOYMENT_TARGET" ] && DT_MINOR=0 +FLOOR_RANK=$(( DT_MAJOR * 1000 + DT_MINOR )) + +echo " deployment target: $DEPLOYMENT_TARGET (rank $FLOOR_RANK)" + +# Pick an iPhone simulator at or above the deployment target. A runtime BELOW it +# is useless — the app cannot install there — which is why this filters rather +# than just taking the oldest installed runtime. +select_sim() { + local which="$1" + xcrun simctl list devices available --json \ + | jq -c --argjson floor "$FLOOR_RANK" --arg which "$which" ' + [ .devices | to_entries[] + | (.key | capture("SimRuntime\\.iOS-(?[0-9]+)-(?[0-9]+)$")) as $v + | (($v.maj | tonumber) * 1000 + ($v.min | tonumber)) as $rank + | select($rank >= $floor) + | .value[] + | select(.name | startswith("iPhone")) + | { rank: $rank, udid: .udid, name: .name, os: "\($v.maj).\($v.min)" } + ] + | sort_by(.rank, .name) + | if length == 0 then empty + elif $which == "floor" then first + else last end + ' +} + +run_tier() { + local which="$1" + local sim udid label + + sim=$(select_sim "$which") + if [ -z "$sim" ]; then + echo "error: no iPhone simulator at or above iOS $DEPLOYMENT_TARGET installed" >&2 + echo "hint: install one with 'xcodebuild -downloadPlatform iOS'" >&2 + return 1 + fi + + udid=$(echo "$sim" | jq -r .udid) + label=$(echo "$sim" | jq -r '"\(.name) (iOS \(.os))"') + + echo + echo "==> $which: $label" + + if [ "$which" = "floor" ] && [ "$(echo "$sim" | jq -r .rank)" -ge $(( (DT_MAJOR + 1) * 1000 )) ]; then + echo " NOTE: nearest installed runtime is a major version above the $DEPLOYMENT_TARGET" + echo " deployment target, so this is not true floor coverage." + fi + + # Findings are printed by the suite itself; surface them plus the verdict. + set -o pipefail + xcodebuild test \ + -project "$PROJECT" \ + -scheme "$SCHEME" \ + -destination "id=$udid" \ + CODE_SIGNING_ALLOWED=NO 2>&1 \ + | grep -E 'finding\(s\)|no accessibility findings|^ • |did not complete in time|Executed [0-9]+ test|\*\* TEST (SUCCEEDED|FAILED)' \ + || true + + return 0 +} + +status=0 +if [ "$TIER" = "both" ]; then + run_tier floor || status=1 + run_tier current || status=1 +else + run_tier "$TIER" || status=1 +fi + +echo +if [ "$status" -eq 0 ]; then + echo "==> Done. Findings above are the burndown list for issue #21." + echo " They are reported, not enforced — widen" + echo " AccessibilityAuditHarness.enforcedAuditTypes as each phase lands." +fi +exit "$status" -- cgit v1.2.3