diff options
| author | Christian Cleberg <[email protected]> | 2026-07-20 17:38:46 -0500 |
|---|---|---|
| committer | Christian Cleberg <[email protected]> | 2026-07-20 17:55:24 -0500 |
| commit | ea57052e998007dc914257f84f383c63f945cf08 (patch) | |
| tree | 0863b3a7180b2807d9709193ca0b382a8b97281c /DomainDigUITests/AccessibilityAuditTests.swift | |
| parent | 38723030eb0db58a9d62a45c7725861901309a05 (diff) | |
| download | domain-dig-ea57052e998007dc914257f84f383c63f945cf08.tar.gz domain-dig-ea57052e998007dc914257f84f383c63f945cf08.tar.bz2 domain-dig-ea57052e998007dc914257f84f383c63f945cf08.zip | |
fix(a11y): survive audit timeouts, and stop overclaiming CI floor coverage
Two problems the first CI run exposed.
Audit timeouts. Three tests failed with "Audit failed to complete in
time" (code -56) on the GitHub runner. That is the audit's own internal
deadline on a slower machine, not an app defect, and the harness had no
resilience to it. Audits now retry up to three times, and a screen that
still cannot be audited is reported via XCTSkip rather than passing.
Skips are distinct from passes in CI, so an unaudited screen stays
visible instead of being silently counted as clean. The Dynamic Type
sweep attempts every screen before skipping, so one slow screen cannot
drop the other four.
Overclaimed floor coverage. The two-simulator matrix was justified on
covering the oldest supported OS, but the macos-26 image ships only iOS
26.x runtimes, so "floor" resolved to 26.2 and "current" to 26.5 — the
run compared two 26.x images and never touched an 18.x one. The measured
non-nested coverage that motivated the matrix (18.6 vs 27.0) reproduces
locally but not on this runner. The workflow comment now states this
plainly, and the selection step emits a warning annotation when the
resolved floor sits a major version or more above the deployment target,
so the gap is visible in the CI UI rather than assumed away.
Installing an older runtime in CI is possible via xcodebuild
-downloadPlatform but costs several GB and minutes per job; left out
pending a call on whether that trade is worth it.
Diffstat (limited to 'DomainDigUITests/AccessibilityAuditTests.swift')
| -rw-r--r-- | DomainDigUITests/AccessibilityAuditTests.swift | 51 |
1 files changed, 30 insertions, 21 deletions
diff --git a/DomainDigUITests/AccessibilityAuditTests.swift b/DomainDigUITests/AccessibilityAuditTests.swift index 63b3d02..6a90bac 100644 --- a/DomainDigUITests/AccessibilityAuditTests.swift +++ b/DomainDigUITests/AccessibilityAuditTests.swift @@ -17,33 +17,23 @@ final class AccessibilityAuditTests: XCTestCase { // MARK: Per-screen audits func testInspectScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Inspect") - try AccessibilityAuditHarness.audit(app, screen: "inspect", test: self) + try auditRootTab("Inspect") } func testDashboardScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Dashboard") - try AccessibilityAuditHarness.audit(app, screen: "dashboard", test: self) + try auditRootTab("Dashboard") } func testAuditScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Audit") - try AccessibilityAuditHarness.audit(app, screen: "audit", test: self) + try auditRootTab("Audit") } func testHistoryScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("History") - try AccessibilityAuditHarness.audit(app, screen: "history", test: self) + try auditRootTab("History") } func testSettingsScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Settings") - try AccessibilityAuditHarness.audit(app, screen: "settings", test: self) + try auditRootTab("Settings") } func testTrackedDomainsScreen() throws { @@ -57,7 +47,8 @@ final class AccessibilityAuditTests: XCTestCase { ) trackedDomains.tap() - try AccessibilityAuditHarness.audit(app, screen: "tracked-domains", test: self) + let audited = try AccessibilityAuditHarness.audit(app, screen: "tracked-domains", test: self) + try XCTSkipUnless(audited, "Tracked Domains audit did not complete in time") } // MARK: Dynamic Type @@ -67,18 +58,36 @@ final class AccessibilityAuditTests: XCTestCase { /// This is where clipped text and fixed-height containers surface — the /// `.accessibility5`-class failures that the fixed geometry in /// `AppDensityMetrics` is expected to produce until phase 3 of #21 lands. + /// + /// Every screen is attempted even if an earlier one times out, so one slow + /// screen cannot silently drop the rest; the skip is reported at the end. func testAllScreensAtLargestAccessibilitySize() throws { let app = AccessibilityAuditHarness.launch( contentSizeCategory: "UICTContentSizeCategoryAccessibilityXXXL" ) + var unaudited: [String] = [] + for tab in ["Inspect", "Dashboard", "Audit", "History", "Settings"] { app.selectRootTab(tab) - try AccessibilityAuditHarness.audit( - app, - screen: "\(tab.lowercased())-accessibilityXXXL", - test: self - ) + let screen = "\(tab.lowercased())-accessibilityXXXL" + if try !AccessibilityAuditHarness.audit(app, screen: screen, test: self) { + unaudited.append(tab) + } } + + try XCTSkipUnless( + unaudited.isEmpty, + "Audit did not complete in time for: \(unaudited.joined(separator: ", "))" + ) + } + + // MARK: Helpers + + private func auditRootTab(_ tab: String) throws { + let app = AccessibilityAuditHarness.launch() + app.selectRootTab(tab) + let audited = try AccessibilityAuditHarness.audit(app, screen: tab.lowercased(), test: self) + try XCTSkipUnless(audited, "Audit did not complete in time for \(tab)") } } |
