diff options
Diffstat (limited to 'DomainDigUITests')
| -rw-r--r-- | DomainDigUITests/AccessibilityAuditHarness.swift | 61 | ||||
| -rw-r--r-- | DomainDigUITests/AccessibilityAuditTests.swift | 51 |
2 files changed, 84 insertions, 28 deletions
diff --git a/DomainDigUITests/AccessibilityAuditHarness.swift b/DomainDigUITests/AccessibilityAuditHarness.swift index 80257e1..9767d95 100644 --- a/DomainDigUITests/AccessibilityAuditHarness.swift +++ b/DomainDigUITests/AccessibilityAuditHarness.swift @@ -40,6 +40,9 @@ enum AccessibilityAuditHarness { /// `.sufficientElementDescription`, `.trait` static let enforcedAuditTypes: XCUIAccessibilityAuditType = [] + /// How many times to retry an audit that misses its internal deadline. + private static let auditAttempts = 3 + /// Launches the app with feature gating lifted, optionally at a specific /// content size category. static func launch(contentSizeCategory: String? = nil) -> XCUIApplication { @@ -56,19 +59,53 @@ enum AccessibilityAuditHarness { /// /// Findings are logged and attached to the result bundle so a CI run /// produces the burndown list as an artifact rather than only a pass/fail. + /// + /// Returns `false` if the audit could not complete, leaving the screen + /// unaudited. Callers turn that into an `XCTSkip` — reporting a pass would + /// claim coverage that did not happen. + @discardableResult static func audit( _ app: XCUIApplication, screen: String, test: XCTestCase - ) throws { + ) throws -> Bool { var findings: [String] = [] + var timeout: Error? + + // The audit traverses the whole element tree and has its own internal + // deadline, which slower CI runners miss on the denser screens. That is a + // tooling timeout, not an app defect, so retry before giving up. + // + // Only the timeout is retried. If a category is enforced and the audit + // reports findings before timing out, those failures are already recorded + // and a retry would duplicate them — accepted, because the alternative is + // losing the run to an infrastructure hiccup. + for attempt in 1...auditAttempts { + findings.removeAll() + timeout = nil + do { + try app.performAccessibilityAudit { issue in + let isEnforced = !enforcedAuditTypes.intersection(issue.auditType).isEmpty + let marker = isEnforced ? "FAIL" : "report" + findings.append("[\(marker)][\(name(for: issue.auditType))] \(issue.compactDescription)") + // true suppresses the finding, false reports it as a test failure. + return !isEnforced + } + break + } catch let error as NSError where error.isAccessibilityAuditTimeout { + timeout = error + print("\(screen): audit timed out (attempt \(attempt) of \(auditAttempts))") + } + } - try app.performAccessibilityAudit { issue in - let isEnforced = !enforcedAuditTypes.intersection(issue.auditType).isEmpty - let marker = isEnforced ? "FAIL" : "report" - findings.append("[\(marker)][\(name(for: issue.auditType))] \(issue.compactDescription)") - // true suppresses the finding, false reports it as a test failure. - return !isEnforced + if timeout != nil { + let message = "\(screen): audit did not complete in time after \(auditAttempts) attempts — screen NOT audited" + print(message) + let attachment = XCTAttachment(string: message) + attachment.name = "a11y-audit-\(screen)-timeout" + attachment.lifetime = .keepAlways + test.add(attachment) + return false } let summary = findings.isEmpty @@ -81,6 +118,8 @@ enum AccessibilityAuditHarness { attachment.name = "a11y-audit-\(screen)" attachment.lifetime = .keepAlways test.add(attachment) + + return true } /// `XCUIAccessibilityAuditType` is an option set whose description is just a @@ -102,6 +141,14 @@ enum AccessibilityAuditHarness { } } +private extension NSError { + /// `Audit failed to complete in time` — the audit's own deadline, raised by + /// XCTest rather than by anything wrong with the app. + var isAccessibilityAuditTimeout: Bool { + domain == "com.apple.xcode.xctest.accessibilityAudit" && code == -56 + } +} + extension XCUIApplication { /// Taps a root tab by its visible label. /// diff --git a/DomainDigUITests/AccessibilityAuditTests.swift b/DomainDigUITests/AccessibilityAuditTests.swift index 63b3d02..6a90bac 100644 --- a/DomainDigUITests/AccessibilityAuditTests.swift +++ b/DomainDigUITests/AccessibilityAuditTests.swift @@ -17,33 +17,23 @@ final class AccessibilityAuditTests: XCTestCase { // MARK: Per-screen audits func testInspectScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Inspect") - try AccessibilityAuditHarness.audit(app, screen: "inspect", test: self) + try auditRootTab("Inspect") } func testDashboardScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Dashboard") - try AccessibilityAuditHarness.audit(app, screen: "dashboard", test: self) + try auditRootTab("Dashboard") } func testAuditScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Audit") - try AccessibilityAuditHarness.audit(app, screen: "audit", test: self) + try auditRootTab("Audit") } func testHistoryScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("History") - try AccessibilityAuditHarness.audit(app, screen: "history", test: self) + try auditRootTab("History") } func testSettingsScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Settings") - try AccessibilityAuditHarness.audit(app, screen: "settings", test: self) + try auditRootTab("Settings") } func testTrackedDomainsScreen() throws { @@ -57,7 +47,8 @@ final class AccessibilityAuditTests: XCTestCase { ) trackedDomains.tap() - try AccessibilityAuditHarness.audit(app, screen: "tracked-domains", test: self) + let audited = try AccessibilityAuditHarness.audit(app, screen: "tracked-domains", test: self) + try XCTSkipUnless(audited, "Tracked Domains audit did not complete in time") } // MARK: Dynamic Type @@ -67,18 +58,36 @@ final class AccessibilityAuditTests: XCTestCase { /// This is where clipped text and fixed-height containers surface — the /// `.accessibility5`-class failures that the fixed geometry in /// `AppDensityMetrics` is expected to produce until phase 3 of #21 lands. + /// + /// Every screen is attempted even if an earlier one times out, so one slow + /// screen cannot silently drop the rest; the skip is reported at the end. func testAllScreensAtLargestAccessibilitySize() throws { let app = AccessibilityAuditHarness.launch( contentSizeCategory: "UICTContentSizeCategoryAccessibilityXXXL" ) + var unaudited: [String] = [] + for tab in ["Inspect", "Dashboard", "Audit", "History", "Settings"] { app.selectRootTab(tab) - try AccessibilityAuditHarness.audit( - app, - screen: "\(tab.lowercased())-accessibilityXXXL", - test: self - ) + let screen = "\(tab.lowercased())-accessibilityXXXL" + if try !AccessibilityAuditHarness.audit(app, screen: screen, test: self) { + unaudited.append(tab) + } } + + try XCTSkipUnless( + unaudited.isEmpty, + "Audit did not complete in time for: \(unaudited.joined(separator: ", "))" + ) + } + + // MARK: Helpers + + private func auditRootTab(_ tab: String) throws { + let app = AccessibilityAuditHarness.launch() + app.selectRootTab(tab) + let audited = try AccessibilityAuditHarness.audit(app, screen: tab.lowercased(), test: self) + try XCTSkipUnless(audited, "Audit did not complete in time for \(tab)") } } |