From ea57052e998007dc914257f84f383c63f945cf08 Mon Sep 17 00:00:00 2001 From: Christian Cleberg Date: Mon, 20 Jul 2026 17:38:46 -0500 Subject: fix(a11y): survive audit timeouts, and stop overclaiming CI floor coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two problems the first CI run exposed. Audit timeouts. Three tests failed with "Audit failed to complete in time" (code -56) on the GitHub runner. That is the audit's own internal deadline on a slower machine, not an app defect, and the harness had no resilience to it. Audits now retry up to three times, and a screen that still cannot be audited is reported via XCTSkip rather than passing. Skips are distinct from passes in CI, so an unaudited screen stays visible instead of being silently counted as clean. The Dynamic Type sweep attempts every screen before skipping, so one slow screen cannot drop the other four. Overclaimed floor coverage. The two-simulator matrix was justified on covering the oldest supported OS, but the macos-26 image ships only iOS 26.x runtimes, so "floor" resolved to 26.2 and "current" to 26.5 — the run compared two 26.x images and never touched an 18.x one. The measured non-nested coverage that motivated the matrix (18.6 vs 27.0) reproduces locally but not on this runner. The workflow comment now states this plainly, and the selection step emits a warning annotation when the resolved floor sits a major version or more above the deployment target, so the gap is visible in the CI UI rather than assumed away. Installing an older runtime in CI is possible via xcodebuild -downloadPlatform but costs several GB and minutes per job; left out pending a call on whether that trade is worth it. --- DomainDigUITests/AccessibilityAuditTests.swift | 51 +++++++++++++++----------- 1 file changed, 30 insertions(+), 21 deletions(-) (limited to 'DomainDigUITests/AccessibilityAuditTests.swift') diff --git a/DomainDigUITests/AccessibilityAuditTests.swift b/DomainDigUITests/AccessibilityAuditTests.swift index 63b3d02..6a90bac 100644 --- a/DomainDigUITests/AccessibilityAuditTests.swift +++ b/DomainDigUITests/AccessibilityAuditTests.swift @@ -17,33 +17,23 @@ final class AccessibilityAuditTests: XCTestCase { // MARK: Per-screen audits func testInspectScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Inspect") - try AccessibilityAuditHarness.audit(app, screen: "inspect", test: self) + try auditRootTab("Inspect") } func testDashboardScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Dashboard") - try AccessibilityAuditHarness.audit(app, screen: "dashboard", test: self) + try auditRootTab("Dashboard") } func testAuditScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Audit") - try AccessibilityAuditHarness.audit(app, screen: "audit", test: self) + try auditRootTab("Audit") } func testHistoryScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("History") - try AccessibilityAuditHarness.audit(app, screen: "history", test: self) + try auditRootTab("History") } func testSettingsScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Settings") - try AccessibilityAuditHarness.audit(app, screen: "settings", test: self) + try auditRootTab("Settings") } func testTrackedDomainsScreen() throws { @@ -57,7 +47,8 @@ final class AccessibilityAuditTests: XCTestCase { ) trackedDomains.tap() - try AccessibilityAuditHarness.audit(app, screen: "tracked-domains", test: self) + let audited = try AccessibilityAuditHarness.audit(app, screen: "tracked-domains", test: self) + try XCTSkipUnless(audited, "Tracked Domains audit did not complete in time") } // MARK: Dynamic Type @@ -67,18 +58,36 @@ final class AccessibilityAuditTests: XCTestCase { /// This is where clipped text and fixed-height containers surface — the /// `.accessibility5`-class failures that the fixed geometry in /// `AppDensityMetrics` is expected to produce until phase 3 of #21 lands. + /// + /// Every screen is attempted even if an earlier one times out, so one slow + /// screen cannot silently drop the rest; the skip is reported at the end. func testAllScreensAtLargestAccessibilitySize() throws { let app = AccessibilityAuditHarness.launch( contentSizeCategory: "UICTContentSizeCategoryAccessibilityXXXL" ) + var unaudited: [String] = [] + for tab in ["Inspect", "Dashboard", "Audit", "History", "Settings"] { app.selectRootTab(tab) - try AccessibilityAuditHarness.audit( - app, - screen: "\(tab.lowercased())-accessibilityXXXL", - test: self - ) + let screen = "\(tab.lowercased())-accessibilityXXXL" + if try !AccessibilityAuditHarness.audit(app, screen: screen, test: self) { + unaudited.append(tab) + } } + + try XCTSkipUnless( + unaudited.isEmpty, + "Audit did not complete in time for: \(unaudited.joined(separator: ", "))" + ) + } + + // MARK: Helpers + + private func auditRootTab(_ tab: String) throws { + let app = AccessibilityAuditHarness.launch() + app.selectRootTab(tab) + let audited = try AccessibilityAuditHarness.audit(app, screen: tab.lowercased(), test: self) + try XCTSkipUnless(audited, "Audit did not complete in time for \(tab)") } } -- cgit v1.2.3