summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristian Cleberg <[email protected]>2026-07-20 17:38:46 -0500
committerChristian Cleberg <[email protected]>2026-07-20 17:55:24 -0500
commitea57052e998007dc914257f84f383c63f945cf08 (patch)
tree0863b3a7180b2807d9709193ca0b382a8b97281c
parent38723030eb0db58a9d62a45c7725861901309a05 (diff)
downloaddomain-dig-ea57052e998007dc914257f84f383c63f945cf08.tar.gz
domain-dig-ea57052e998007dc914257f84f383c63f945cf08.tar.bz2
domain-dig-ea57052e998007dc914257f84f383c63f945cf08.zip
fix(a11y): survive audit timeouts, and stop overclaiming CI floor coverage
Two problems the first CI run exposed. Audit timeouts. Three tests failed with "Audit failed to complete in time" (code -56) on the GitHub runner. That is the audit's own internal deadline on a slower machine, not an app defect, and the harness had no resilience to it. Audits now retry up to three times, and a screen that still cannot be audited is reported via XCTSkip rather than passing. Skips are distinct from passes in CI, so an unaudited screen stays visible instead of being silently counted as clean. The Dynamic Type sweep attempts every screen before skipping, so one slow screen cannot drop the other four. Overclaimed floor coverage. The two-simulator matrix was justified on covering the oldest supported OS, but the macos-26 image ships only iOS 26.x runtimes, so "floor" resolved to 26.2 and "current" to 26.5 — the run compared two 26.x images and never touched an 18.x one. The measured non-nested coverage that motivated the matrix (18.6 vs 27.0) reproduces locally but not on this runner. The workflow comment now states this plainly, and the selection step emits a warning annotation when the resolved floor sits a major version or more above the deployment target, so the gap is visible in the CI UI rather than assumed away. Installing an older runtime in CI is possible via xcodebuild -downloadPlatform but costs several GB and minutes per job; left out pending a call on whether that trade is worth it.
-rw-r--r--.github/workflows/build.yml41
-rw-r--r--DomainDigUITests/AccessibilityAuditHarness.swift61
-rw-r--r--DomainDigUITests/AccessibilityAuditTests.swift51
3 files changed, 114 insertions, 39 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index a6a2113..e0c2eee 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -20,18 +20,28 @@ name: Build
# TEST_RUNNER_-prefixed build setting reaches the UI test process.)
#
# The matrix runs two simulators because audit coverage is NOT nested — each
-# runtime reports findings the other misses, in both directions. Measured on the
-# Tracked Domains screen, iOS 18.6 reported 2 issues and iOS 27.0 reported 6
-# (including contrast and element-detection issues 18.6 never raised); at
-# accessibility text sizes the Dashboard produced a hit-region finding on 18.6
-# that 27.0 did not. Testing only the newest image would leave the oldest
-# supported OS unchecked; testing only the floor would miss newer audit checks.
+# runtime reports findings the other misses, in both directions. Measured
+# locally on the Tracked Domains screen, iOS 18.6 reported 2 issues and iOS 27.0
+# reported 6 (including contrast and element-detection issues 18.6 never
+# raised); at accessibility text sizes the Dashboard produced a hit-region
+# finding on 18.6 that 27.0 did not.
#
-# Runtimes are resolved dynamically rather than pinned: the deployment target is
-# 17.6, but no 17.6 simulator runtime ships, so "floor" means the oldest
-# available runtime at or above the deployment target (18.6 at time of writing).
-# The previous selector took the first iPhone from any runtime, which could pick
-# a simulator BELOW the deployment target, where the app cannot install.
+# CAVEAT — this image cannot test the real support floor. The app's deployment
+# target is 17.6, but the macos-26 runner ships only iOS 26.x simulator
+# runtimes, so "floor" resolves to ~26.2 here rather than an 18.x image. CI
+# therefore compares two 26.x runtimes; genuine oldest-supported-OS coverage has
+# to come from a local run or a self-hosted runner with older runtimes
+# installed. The "Select simulator" step emits a warning annotation when the
+# resolved floor sits well above the deployment target, so this gap stays
+# visible instead of being silently assumed away.
+#
+# Installing an older runtime in CI (xcodebuild -downloadPlatform iOS
+# -buildVersion 18.6) is possible but costs several GB and minutes per job; not
+# done by default.
+#
+# Runtimes are resolved dynamically rather than pinned. The previous selector
+# took the first iPhone from any runtime, which could pick a simulator BELOW the
+# deployment target, where the app cannot install.
#
# pull_request only, plus manual dispatch. GitHub builds the merge result (PR
# merged into main), so a green PR validates exactly what will land on main.
@@ -115,6 +125,15 @@ jobs:
echo "udid=$(echo "$selected" | jq -r .udid)" >> "$GITHUB_OUTPUT"
echo "label=$label" >> "$GITHUB_OUTPUT"
+ # Surface the floor-coverage gap rather than letting the matrix imply
+ # coverage it does not have. One major version of slack is tolerated.
+ if [ "${{ matrix.tier }}" = "floor" ]; then
+ rank=$(echo "$selected" | jq -r .rank)
+ if [ "$rank" -ge $(( (DEPLOYMENT_TARGET_MAJOR + 1) * 1000 )) ]; then
+ echo "::warning::Floor tier resolved to $label, well above the ${DEPLOYMENT_TARGET_MAJOR}.${DEPLOYMENT_TARGET_MINOR} deployment target. This image has no older runtime, so the oldest supported OS is NOT covered by this run."
+ fi
+ fi
+
- name: Test on ${{ steps.sim.outputs.label }}
run: |
set -o pipefail
diff --git a/DomainDigUITests/AccessibilityAuditHarness.swift b/DomainDigUITests/AccessibilityAuditHarness.swift
index 80257e1..9767d95 100644
--- a/DomainDigUITests/AccessibilityAuditHarness.swift
+++ b/DomainDigUITests/AccessibilityAuditHarness.swift
@@ -40,6 +40,9 @@ enum AccessibilityAuditHarness {
/// `.sufficientElementDescription`, `.trait`
static let enforcedAuditTypes: XCUIAccessibilityAuditType = []
+ /// How many times to retry an audit that misses its internal deadline.
+ private static let auditAttempts = 3
+
/// Launches the app with feature gating lifted, optionally at a specific
/// content size category.
static func launch(contentSizeCategory: String? = nil) -> XCUIApplication {
@@ -56,19 +59,53 @@ enum AccessibilityAuditHarness {
///
/// Findings are logged and attached to the result bundle so a CI run
/// produces the burndown list as an artifact rather than only a pass/fail.
+ ///
+ /// Returns `false` if the audit could not complete, leaving the screen
+ /// unaudited. Callers turn that into an `XCTSkip` — reporting a pass would
+ /// claim coverage that did not happen.
+ @discardableResult
static func audit(
_ app: XCUIApplication,
screen: String,
test: XCTestCase
- ) throws {
+ ) throws -> Bool {
var findings: [String] = []
+ var timeout: Error?
+
+ // The audit traverses the whole element tree and has its own internal
+ // deadline, which slower CI runners miss on the denser screens. That is a
+ // tooling timeout, not an app defect, so retry before giving up.
+ //
+ // Only the timeout is retried. If a category is enforced and the audit
+ // reports findings before timing out, those failures are already recorded
+ // and a retry would duplicate them — accepted, because the alternative is
+ // losing the run to an infrastructure hiccup.
+ for attempt in 1...auditAttempts {
+ findings.removeAll()
+ timeout = nil
+ do {
+ try app.performAccessibilityAudit { issue in
+ let isEnforced = !enforcedAuditTypes.intersection(issue.auditType).isEmpty
+ let marker = isEnforced ? "FAIL" : "report"
+ findings.append("[\(marker)][\(name(for: issue.auditType))] \(issue.compactDescription)")
+ // true suppresses the finding, false reports it as a test failure.
+ return !isEnforced
+ }
+ break
+ } catch let error as NSError where error.isAccessibilityAuditTimeout {
+ timeout = error
+ print("\(screen): audit timed out (attempt \(attempt) of \(auditAttempts))")
+ }
+ }
- try app.performAccessibilityAudit { issue in
- let isEnforced = !enforcedAuditTypes.intersection(issue.auditType).isEmpty
- let marker = isEnforced ? "FAIL" : "report"
- findings.append("[\(marker)][\(name(for: issue.auditType))] \(issue.compactDescription)")
- // true suppresses the finding, false reports it as a test failure.
- return !isEnforced
+ if timeout != nil {
+ let message = "\(screen): audit did not complete in time after \(auditAttempts) attempts — screen NOT audited"
+ print(message)
+ let attachment = XCTAttachment(string: message)
+ attachment.name = "a11y-audit-\(screen)-timeout"
+ attachment.lifetime = .keepAlways
+ test.add(attachment)
+ return false
}
let summary = findings.isEmpty
@@ -81,6 +118,8 @@ enum AccessibilityAuditHarness {
attachment.name = "a11y-audit-\(screen)"
attachment.lifetime = .keepAlways
test.add(attachment)
+
+ return true
}
/// `XCUIAccessibilityAuditType` is an option set whose description is just a
@@ -102,6 +141,14 @@ enum AccessibilityAuditHarness {
}
}
+private extension NSError {
+ /// `Audit failed to complete in time` — the audit's own deadline, raised by
+ /// XCTest rather than by anything wrong with the app.
+ var isAccessibilityAuditTimeout: Bool {
+ domain == "com.apple.xcode.xctest.accessibilityAudit" && code == -56
+ }
+}
+
extension XCUIApplication {
/// Taps a root tab by its visible label.
///
diff --git a/DomainDigUITests/AccessibilityAuditTests.swift b/DomainDigUITests/AccessibilityAuditTests.swift
index 63b3d02..6a90bac 100644
--- a/DomainDigUITests/AccessibilityAuditTests.swift
+++ b/DomainDigUITests/AccessibilityAuditTests.swift
@@ -17,33 +17,23 @@ final class AccessibilityAuditTests: XCTestCase {
// MARK: Per-screen audits
func testInspectScreen() throws {
- let app = AccessibilityAuditHarness.launch()
- app.selectRootTab("Inspect")
- try AccessibilityAuditHarness.audit(app, screen: "inspect", test: self)
+ try auditRootTab("Inspect")
}
func testDashboardScreen() throws {
- let app = AccessibilityAuditHarness.launch()
- app.selectRootTab("Dashboard")
- try AccessibilityAuditHarness.audit(app, screen: "dashboard", test: self)
+ try auditRootTab("Dashboard")
}
func testAuditScreen() throws {
- let app = AccessibilityAuditHarness.launch()
- app.selectRootTab("Audit")
- try AccessibilityAuditHarness.audit(app, screen: "audit", test: self)
+ try auditRootTab("Audit")
}
func testHistoryScreen() throws {
- let app = AccessibilityAuditHarness.launch()
- app.selectRootTab("History")
- try AccessibilityAuditHarness.audit(app, screen: "history", test: self)
+ try auditRootTab("History")
}
func testSettingsScreen() throws {
- let app = AccessibilityAuditHarness.launch()
- app.selectRootTab("Settings")
- try AccessibilityAuditHarness.audit(app, screen: "settings", test: self)
+ try auditRootTab("Settings")
}
func testTrackedDomainsScreen() throws {
@@ -57,7 +47,8 @@ final class AccessibilityAuditTests: XCTestCase {
)
trackedDomains.tap()
- try AccessibilityAuditHarness.audit(app, screen: "tracked-domains", test: self)
+ let audited = try AccessibilityAuditHarness.audit(app, screen: "tracked-domains", test: self)
+ try XCTSkipUnless(audited, "Tracked Domains audit did not complete in time")
}
// MARK: Dynamic Type
@@ -67,18 +58,36 @@ final class AccessibilityAuditTests: XCTestCase {
/// This is where clipped text and fixed-height containers surface — the
/// `.accessibility5`-class failures that the fixed geometry in
/// `AppDensityMetrics` is expected to produce until phase 3 of #21 lands.
+ ///
+ /// Every screen is attempted even if an earlier one times out, so one slow
+ /// screen cannot silently drop the rest; the skip is reported at the end.
func testAllScreensAtLargestAccessibilitySize() throws {
let app = AccessibilityAuditHarness.launch(
contentSizeCategory: "UICTContentSizeCategoryAccessibilityXXXL"
)
+ var unaudited: [String] = []
+
for tab in ["Inspect", "Dashboard", "Audit", "History", "Settings"] {
app.selectRootTab(tab)
- try AccessibilityAuditHarness.audit(
- app,
- screen: "\(tab.lowercased())-accessibilityXXXL",
- test: self
- )
+ let screen = "\(tab.lowercased())-accessibilityXXXL"
+ if try !AccessibilityAuditHarness.audit(app, screen: screen, test: self) {
+ unaudited.append(tab)
+ }
}
+
+ try XCTSkipUnless(
+ unaudited.isEmpty,
+ "Audit did not complete in time for: \(unaudited.joined(separator: ", "))"
+ )
+ }
+
+ // MARK: Helpers
+
+ private func auditRootTab(_ tab: String) throws {
+ let app = AccessibilityAuditHarness.launch()
+ app.selectRootTab(tab)
+ let audited = try AccessibilityAuditHarness.audit(app, screen: tab.lowercased(), test: self)
+ try XCTSkipUnless(audited, "Audit did not complete in time for \(tab)")
}
}