diff options
| -rwxr-xr-x | .githooks/pre-push | 68 | ||||
| -rw-r--r-- | .github/workflows/build.yml | 92 | ||||
| -rw-r--r-- | Docs/ACCESSIBILITY.md | 96 | ||||
| -rw-r--r-- | README.md | 24 | ||||
| -rwxr-xr-x | Scripts/audit-a11y.sh | 129 |
5 files changed, 348 insertions, 61 deletions
diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100755 index 0000000..aff4368 --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# +# Run the accessibility audit against the oldest supported simulator runtime +# before pushing. +# +# Enable once per clone: +# git config core.hooksPath .githooks +# +# Why pre-push and not pre-commit: the suite takes ~85s. At pre-commit that +# blocks every commit, and a hook you routinely bypass with --no-verify is worse +# than no hook, because it trains you to ignore it. Pushes are far less frequent +# and map to the unit of work that actually reaches CI. +# +# Why only the floor tier: CI already audits the newest runtime on a clean +# checkout. The GitHub image has no old runtimes, so the floor is the one thing +# CI structurally cannot cover — and it is the one this machine can. Running +# both here would just double the wait to re-check what CI already does. +# +# Skip deliberately with: git push --no-verify + +set -euo pipefail + +repo_root=$(git rev-parse --show-toplevel) +cd "$repo_root" + +zero='0000000000000000000000000000000000000000' +changed='' + +# stdin: <local ref> <local sha> <remote ref> <remote sha>, one line per ref. +while read -r _local_ref local_sha _remote_ref remote_sha; do + [ "$local_sha" = "$zero" ] && continue # branch deletion + + if [ "$remote_sha" = "$zero" ]; then + # New branch: diff against the default branch rather than the whole history. + base=$(git merge-base origin/main "$local_sha" 2>/dev/null || echo '') + range="${base:+$base..}$local_sha" + else + range="$remote_sha..$local_sha" + fi + + changed="$changed$(git diff --name-only "$range" 2>/dev/null || true)"$'\n' +done + +if [ -z "$(printf '%s' "$changed" | tr -d '[:space:]')" ]; then + exit 0 +fi + +# Only pay the ~85s when something could actually change the rendered UI. +if ! printf '%s' "$changed" | grep -qE '\.(swift|xcassets|xcodeproj)|\.pbxproj|xcscheme'; then + echo "pre-push: no Swift/project changes, skipping accessibility audit" + exit 0 +fi + +echo "pre-push: running accessibility audit on the floor runtime (~85s)" +echo " skip with 'git push --no-verify'" + +if ! ./Scripts/audit-a11y.sh floor; then + echo + echo "pre-push: audit could not run. Push aborted." >&2 + echo " Re-run with './Scripts/audit-a11y.sh floor' to see why," >&2 + echo " or bypass with 'git push --no-verify'." >&2 + exit 1 +fi + +# Note: findings are reported, not enforced, so a clean exit here does not mean +# zero findings — read the list above. Enforcement is controlled by +# AccessibilityAuditHarness.enforcedAuditTypes. +exit 0 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e0c2eee..5d9d877 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,39 +9,29 @@ name: Build # dependencies). The test target is DomainDigUITests — an accessibility audit # suite; see DomainDigUITests/AccessibilityAuditHarness.swift. # +# WHAT THIS JOB IS FOR, given the audit also runs locally: +# a clean checkout of the merge result. A local hook runs against the working +# tree and therefore cannot catch a file that was never committed — the failure +# mode that matters most here, since DomainDig.xcodeproj is hand-edited and uses +# file-system-synchronized groups where a whole missing folder still builds fine +# locally. This job is the only place that check exists; sr.ht cannot run it. +# +# DELIBERATELY ONE JOB, NEWEST RUNTIME ONLY. Audit coverage is not nested across +# OS versions, so the oldest supported OS genuinely needs its own run — but the +# macos-26 image ships only iOS 26.x runtimes, so CI *cannot* provide it. Asking +# for two jobs here bought two near-identical 26.x runs at double the macOS +# minutes. Floor coverage lives in Scripts/audit-a11y.sh, run from a machine that +# actually has an 18.x runtime installed, and is wired to the pre-push hook in +# .githooks/. See Docs/ACCESSIBILITY.md for the split. +# # The audit REPORTS but does not FAIL by default. It surfaces violations that # exist today, so gating on it would block every unrelated PR until the # accessibility pass in issue #21 completes. Findings land in the job log and in -# the uploaded .xcresult bundle, tagged [report] or [FAIL]. -# -# Enforcement is a committed constant, not a CI setting: widen -# `AccessibilityAuditHarness.enforcedAuditTypes` as each phase clears a category. -# (Env vars were tried first — neither a plain xcodebuild env var nor a -# TEST_RUNNER_-prefixed build setting reaches the UI test process.) -# -# The matrix runs two simulators because audit coverage is NOT nested — each -# runtime reports findings the other misses, in both directions. Measured -# locally on the Tracked Domains screen, iOS 18.6 reported 2 issues and iOS 27.0 -# reported 6 (including contrast and element-detection issues 18.6 never -# raised); at accessibility text sizes the Dashboard produced a hit-region -# finding on 18.6 that 27.0 did not. -# -# CAVEAT — this image cannot test the real support floor. The app's deployment -# target is 17.6, but the macos-26 runner ships only iOS 26.x simulator -# runtimes, so "floor" resolves to ~26.2 here rather than an 18.x image. CI -# therefore compares two 26.x runtimes; genuine oldest-supported-OS coverage has -# to come from a local run or a self-hosted runner with older runtimes -# installed. The "Select simulator" step emits a warning annotation when the -# resolved floor sits well above the deployment target, so this gap stays -# visible instead of being silently assumed away. -# -# Installing an older runtime in CI (xcodebuild -downloadPlatform iOS -# -buildVersion 18.6) is possible but costs several GB and minutes per job; not -# done by default. -# -# Runtimes are resolved dynamically rather than pinned. The previous selector -# took the first iPhone from any runtime, which could pick a simulator BELOW the -# deployment target, where the app cannot install. +# the uploaded .xcresult bundle, tagged [report] or [FAIL]. Enforcement is a +# committed constant: widen `AccessibilityAuditHarness.enforcedAuditTypes` as +# each phase clears a category. (Env vars were tried first — neither a plain +# xcodebuild env var nor a TEST_RUNNER_-prefixed build setting reaches the UI +# test process.) # # pull_request only, plus manual dispatch. GitHub builds the merge result (PR # merged into main), so a green PR validates exactly what will land on main. @@ -65,25 +55,13 @@ concurrency: group: build-${{ github.ref }} cancel-in-progress: true -env: - # Keep in sync with IPHONEOS_DEPLOYMENT_TARGET in DomainDig.xcodeproj. - DEPLOYMENT_TARGET_MAJOR: '17' - DEPLOYMENT_TARGET_MINOR: '6' - jobs: test: - name: xcodebuild test (${{ matrix.tier }}) + name: xcodebuild test # macos-latest still points at macOS 15, which lacks the iOS 26+ SDK this app # is built against. runs-on: macos-26 - strategy: - fail-fast: false - matrix: - # floor = oldest runtime the app actually supports - # current = newest runtime available on the image - tier: [floor, current] - steps: - uses: actions/checkout@v7 @@ -96,26 +74,27 @@ jobs: id: sim run: | set -euo pipefail - floor=$(( DEPLOYMENT_TARGET_MAJOR * 1000 + DEPLOYMENT_TARGET_MINOR )) + # Newest available iPhone runtime. No deployment-target filtering is + # needed for "newest" — it is always at or above the floor. The + # previous selector took the first iPhone from ANY runtime, which on a + # machine with an older runtime installed could pick a simulator below + # the deployment target, where the app cannot install. selected=$(xcrun simctl list devices available --json \ - | jq -c --argjson floor "$floor" --arg tier "${{ matrix.tier }}" ' + | jq -c ' [ .devices | to_entries[] | (.key | capture("SimRuntime\\.iOS-(?<maj>[0-9]+)-(?<min>[0-9]+)$")) as $v | (($v.maj | tonumber) * 1000 + ($v.min | tonumber)) as $rank - | select($rank >= $floor) | .value[] | select(.name | startswith("iPhone")) | { rank: $rank, udid: .udid, name: .name, os: "\($v.maj).\($v.min)" } ] | sort_by(.rank, .name) - | if length == 0 then empty - elif $tier == "floor" then .[0] - else .[-1] end + | last ') - if [ -z "$selected" ]; then - echo "::error::No iPhone simulator at or above iOS ${DEPLOYMENT_TARGET_MAJOR}.${DEPLOYMENT_TARGET_MINOR} on this image" + if [ -z "$selected" ] || [ "$selected" = "null" ]; then + echo "::error::No iPhone simulator available on this image" xcrun simctl list devices available >&2 exit 1 fi @@ -125,15 +104,6 @@ jobs: echo "udid=$(echo "$selected" | jq -r .udid)" >> "$GITHUB_OUTPUT" echo "label=$label" >> "$GITHUB_OUTPUT" - # Surface the floor-coverage gap rather than letting the matrix imply - # coverage it does not have. One major version of slack is tolerated. - if [ "${{ matrix.tier }}" = "floor" ]; then - rank=$(echo "$selected" | jq -r .rank) - if [ "$rank" -ge $(( (DEPLOYMENT_TARGET_MAJOR + 1) * 1000 )) ]; then - echo "::warning::Floor tier resolved to $label, well above the ${DEPLOYMENT_TARGET_MAJOR}.${DEPLOYMENT_TARGET_MINOR} deployment target. This image has no older runtime, so the oldest supported OS is NOT covered by this run." - fi - fi - - name: Test on ${{ steps.sim.outputs.label }} run: | set -o pipefail @@ -150,6 +120,6 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: test-results-${{ matrix.tier }} + name: test-results path: TestResults.xcresult retention-days: 7 diff --git a/Docs/ACCESSIBILITY.md b/Docs/ACCESSIBILITY.md new file mode 100644 index 0000000..1045df9 --- /dev/null +++ b/Docs/ACCESSIBILITY.md @@ -0,0 +1,96 @@ +# Accessibility Audit + +`DomainDigUITests` runs Apple's `performAccessibilityAudit()` across every +primary screen. The audit checks contrast, hit-region size, clipped text at +large Dynamic Type, element descriptions, trait correctness, and Dynamic Type +support — the same ground the accessibility pass tracked in +[issue #21](https://github.com/zerolabsco/domain-dig/issues/21) covers. + +## Findings are reported, not enforced + +The audit surfaces violations that exist today, so failing on all of them would +block every unrelated change until the whole pass lands. Instead, findings are +logged and attached to the result bundle tagged `[report]` or `[FAIL]`. + +Enforcement is the committed constant +`AccessibilityAuditHarness.enforcedAuditTypes`. Widen it as each phase clears a +category: + +| After phase | Enforce | +| --- | --- | +| 2 — semantic colors + light mode | `.contrast` | +| 3 — Dynamic Type + reflow | `.textClipped`, `.dynamicType`, `.hitRegion` | +| 4 — VoiceOver | `.elementDetection`, `.sufficientElementDescription`, `.trait` | + +A constant rather than a CI setting, for two reasons. Environment variables do +not work: neither a plain `xcodebuild` env var nor a `TEST_RUNNER_`-prefixed +build setting reaches the UI test process, so the toggle silently did nothing. +And a committed value makes "when did contrast become enforced?" answerable with +`git blame` instead of CI tribal knowledge. + +## Why coverage is split between local and CI + +**Audit coverage is not nested across OS versions.** Each runtime reports +findings the others miss, in *both* directions. Measured on this project: + +| Screen | iOS 18.6 | iOS 27.0 | +| --- | --- | --- | +| Tracked Domains | 2 (text clipped) | **6** (+ contrast ×3, element detection) | +| Settings | 2 contrast | **`dynamicType`** finding 18.6 missed | +| Dashboard @ `AccessibilityXXXL` | **hit region** + 2 clipped | 1 clipped only | + +Neither runtime is a superset, so the oldest supported OS needs its own run. +This also rules out committing per-screen baseline counts as a regression guard: +no single number is correct on both. + +The catch is that **GitHub's `macos-26` image ships only iOS 26.x simulator +runtimes.** It cannot test the 17.6 floor at all. A two-job CI matrix was tried +and produced two near-identical 26.x runs at double the macOS minutes. + +So the work is split by what each side can uniquely do: + +| | Runtime | Uniquely provides | +| --- | --- | --- | +| **CI** (`.github/workflows/build.yml`) | newest available | A clean checkout of the merge result — catches a file that was never committed, which a local run cannot. Matters here because `DomainDig.xcodeproj` is hand-edited and uses file-system-synchronized groups, where a whole missing folder still builds locally. | +| **Local** (`Scripts/audit-a11y.sh`) | oldest supported + newest | Real floor coverage, on a machine that actually has an 18.x runtime installed. | + +Together they cover both ends; neither duplicates the other. + +## Running it + +```sh +./Scripts/audit-a11y.sh # floor + current +./Scripts/audit-a11y.sh floor # oldest supported only (~85s) +./Scripts/audit-a11y.sh current # newest installed only +``` + +The script reads the deployment target from the project rather than hard-coding +it, and selects the oldest installed runtime **at or above** it — a runtime +below the deployment target is useless, because the app cannot install there. +If the nearest installed runtime is a major version above the target, it says +so rather than implying floor coverage it does not have. + +### Pre-push hook + +```sh +git config core.hooksPath .githooks +``` + +Runs the floor audit before a push, and only when Swift, asset, or project files +changed. Bypass with `git push --no-verify`. + +Pre-push rather than pre-commit deliberately: the suite takes ~85s, and at +pre-commit that blocks every commit. A hook routinely bypassed with +`--no-verify` is worse than no hook, because it trains you to ignore it. + +## Notes + +- Audits retry up to three times. Slower machines can miss the audit's internal + deadline (`Audit failed to complete in time`, code `-56`), which is a tooling + timeout, not an app defect. A screen that still cannot be audited is reported + as an `XCTSkip`, never a pass — skips are visually distinct in CI, so an + unaudited screen stays visible instead of being silently counted as clean. +- The suite launches with `DOMAIN_DIG_FORCE_PRO_PLUS` so Pro-gated screens are + reachable. `PurchaseService` honours that argument in `DEBUG` builds only. +- Everything used is available at the iOS 17.6 deployment floor; + `performAccessibilityAudit` is `ios(17.0)`. @@ -53,6 +53,30 @@ xcodebuild -project DomainDig.xcodeproj -scheme DomainDig -destination 'platform The app and local API share the canonical report pipeline through `DomainInspectionService`, `DomainReportBuilder`, and `DomainReportExporter`. +### Accessibility Audit + +`DomainDigUITests` runs `performAccessibilityAudit()` over every primary screen, +at default and at the largest accessibility text size. + +```sh +./Scripts/audit-a11y.sh # oldest supported + newest runtime +./Scripts/audit-a11y.sh floor # oldest supported runtime only (~85s) +``` + +Findings are **reported, not enforced** — they are the burndown list for the +accessibility pass. Widen `AccessibilityAuditHarness.enforcedAuditTypes` to turn +a category into a build failure as each phase lands. + +Optional pre-push hook, which runs the floor audit when Swift or project files +change: + +```sh +git config core.hooksPath .githooks +``` + +See [Docs/ACCESSIBILITY.md](Docs/ACCESSIBILITY.md) for why coverage is split +between this script and CI. + ## Release Planning See `RELEASE_ROADMAP.md` for the semver release plan from `v4.4.1` through the planned `v5.0.0` stabilization milestone. diff --git a/Scripts/audit-a11y.sh b/Scripts/audit-a11y.sh new file mode 100755 index 0000000..9ac3aa0 --- /dev/null +++ b/Scripts/audit-a11y.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# +# Run the accessibility audit suite against real simulator runtimes. +# +# ./Scripts/audit-a11y.sh # floor + current +# ./Scripts/audit-a11y.sh floor # oldest supported runtime only +# ./Scripts/audit-a11y.sh current # newest installed runtime only +# +# Why this exists rather than living entirely in CI: audit coverage is NOT +# nested across OS versions — each runtime reports findings the others miss, in +# both directions. Measured on Tracked Domains, iOS 18.6 reported 2 findings and +# iOS 27.0 reported 6; at accessibility text sizes the Dashboard produced a +# hit-region finding on 18.6 that 27.0 did not. The GitHub macos-26 image ships +# only iOS 26.x runtimes, so it structurally cannot cover the floor. This machine +# can. +# +# CI covers "current" on a clean checkout (which a local run cannot, since it +# would miss an uncommitted file). This script covers "floor" (which CI cannot). +# Together they cover both; neither duplicates the other. +# +# The deployment target is read from the project rather than hard-coded, so it +# cannot drift out of sync. + +set -euo pipefail + +cd "$(dirname "$0")/.." + +PROJECT="DomainDig.xcodeproj" +SCHEME="DomainDig" +TIER="${1:-both}" + +case "$TIER" in + floor | current | both) ;; + *) + echo "usage: $0 [floor|current|both]" >&2 + exit 2 + ;; +esac + +echo "==> Reading deployment target from $PROJECT" +DEPLOYMENT_TARGET=$( + xcodebuild -project "$PROJECT" -scheme "$SCHEME" -showBuildSettings 2>/dev/null \ + | awk -F' = ' '/ IPHONEOS_DEPLOYMENT_TARGET = /{print $2; exit}' +) + +if [ -z "${DEPLOYMENT_TARGET:-}" ]; then + echo "error: could not read IPHONEOS_DEPLOYMENT_TARGET" >&2 + exit 1 +fi + +DT_MAJOR="${DEPLOYMENT_TARGET%%.*}" +DT_MINOR="${DEPLOYMENT_TARGET##*.}" +[ "$DT_MINOR" = "$DEPLOYMENT_TARGET" ] && DT_MINOR=0 +FLOOR_RANK=$(( DT_MAJOR * 1000 + DT_MINOR )) + +echo " deployment target: $DEPLOYMENT_TARGET (rank $FLOOR_RANK)" + +# Pick an iPhone simulator at or above the deployment target. A runtime BELOW it +# is useless — the app cannot install there — which is why this filters rather +# than just taking the oldest installed runtime. +select_sim() { + local which="$1" + xcrun simctl list devices available --json \ + | jq -c --argjson floor "$FLOOR_RANK" --arg which "$which" ' + [ .devices | to_entries[] + | (.key | capture("SimRuntime\\.iOS-(?<maj>[0-9]+)-(?<min>[0-9]+)$")) as $v + | (($v.maj | tonumber) * 1000 + ($v.min | tonumber)) as $rank + | select($rank >= $floor) + | .value[] + | select(.name | startswith("iPhone")) + | { rank: $rank, udid: .udid, name: .name, os: "\($v.maj).\($v.min)" } + ] + | sort_by(.rank, .name) + | if length == 0 then empty + elif $which == "floor" then first + else last end + ' +} + +run_tier() { + local which="$1" + local sim udid label + + sim=$(select_sim "$which") + if [ -z "$sim" ]; then + echo "error: no iPhone simulator at or above iOS $DEPLOYMENT_TARGET installed" >&2 + echo "hint: install one with 'xcodebuild -downloadPlatform iOS'" >&2 + return 1 + fi + + udid=$(echo "$sim" | jq -r .udid) + label=$(echo "$sim" | jq -r '"\(.name) (iOS \(.os))"') + + echo + echo "==> $which: $label" + + if [ "$which" = "floor" ] && [ "$(echo "$sim" | jq -r .rank)" -ge $(( (DT_MAJOR + 1) * 1000 )) ]; then + echo " NOTE: nearest installed runtime is a major version above the $DEPLOYMENT_TARGET" + echo " deployment target, so this is not true floor coverage." + fi + + # Findings are printed by the suite itself; surface them plus the verdict. + set -o pipefail + xcodebuild test \ + -project "$PROJECT" \ + -scheme "$SCHEME" \ + -destination "id=$udid" \ + CODE_SIGNING_ALLOWED=NO 2>&1 \ + | grep -E 'finding\(s\)|no accessibility findings|^ • |did not complete in time|Executed [0-9]+ test|\*\* TEST (SUCCEEDED|FAILED)' \ + || true + + return 0 +} + +status=0 +if [ "$TIER" = "both" ]; then + run_tier floor || status=1 + run_tier current || status=1 +else + run_tier "$TIER" || status=1 +fi + +echo +if [ "$status" -eq 0 ]; then + echo "==> Done. Findings above are the burndown list for issue #21." + echo " They are reported, not enforced — widen" + echo " AccessibilityAuditHarness.enforcedAuditTypes as each phase lands." +fi +exit "$status" |