diff options
Diffstat (limited to 'DomainDigUITests/AccessibilityAuditHarness.swift')
| -rw-r--r-- | DomainDigUITests/AccessibilityAuditHarness.swift | 105 |
1 files changed, 80 insertions, 25 deletions
diff --git a/DomainDigUITests/AccessibilityAuditHarness.swift b/DomainDigUITests/AccessibilityAuditHarness.swift index e025dcd..64d0993 100644 --- a/DomainDigUITests/AccessibilityAuditHarness.swift +++ b/DomainDigUITests/AccessibilityAuditHarness.swift @@ -28,17 +28,23 @@ enum AccessibilityAuditHarness { /// reachable. `PurchaseService` honours this in `DEBUG` builds only. private static let forceProPlusArgument = "DOMAIN_DIG_FORCE_PRO_PLUS" - /// Audit categories that fail the build. Everything else is reported only. + /// Audit categories that fail the build on the empty-state suite. A named + /// finding in any of these is a regression in the phase 1–5 work. /// - /// Empty until the accessibility pass starts landing. Suggested ratchet, - /// following the phases in issue #21: - /// - /// - after phase 2 (semantic colors + light mode): `.contrast` - /// - after phase 3 (Dynamic Type + reflow): `.textClipped`, `.dynamicType`, - /// `.hitRegion` - /// - after phase 4 (VoiceOver): `.elementDetection`, - /// `.sufficientElementDescription`, `.trait` - static let enforcedAuditTypes: XCUIAccessibilityAuditType = [] + /// `.contrast` is deliberately absent: the two long-standing Settings + /// findings come from rows scrolled under the translucent tab bar, and their + /// attribution flips between a row name and nil run-to-run, so there is no + /// suppression narrow enough to keep CI stable. Contrast stays report-only, + /// with the palette centralised in `Shared/Colors.xcassets` as the actual + /// guard. + static let enforcedAuditTypes: XCUIAccessibilityAuditType = [ + .textClipped, + .dynamicType, + .hitRegion, + .elementDetection, + .sufficientElementDescription, + .trait + ] /// How many times to retry an audit that misses its internal deadline. private static let auditAttempts = 3 @@ -72,11 +78,18 @@ enum AccessibilityAuditHarness { /// Returns `false` if the audit could not complete, leaving the screen /// unaudited. Callers turn that into an `XCTSkip` — reporting a pass would /// claim coverage that did not happen. + /// `reportOnly` disables enforcement for this call. Used by the seeded + /// tests: bisection showed the audit degrades on `children: .ignore` + /// content — the correct VoiceOver treatment for dense rows — reporting + /// unattributed contrast/dynamicType failures on rows that measure 6–7:1 + /// and render correctly. Until that behaves, the seeded screens report + /// their burndown without gating CI. @discardableResult static func audit( _ app: XCUIApplication, screen: String, - test: XCTestCase + test: XCTestCase, + reportOnly: Bool = false ) throws -> Bool { var findings: [String] = [] var timeout: Error? @@ -94,26 +107,22 @@ enum AccessibilityAuditHarness { timeout = nil do { try app.performAccessibilityAudit { issue in - // WCAG 1.4.3 exempts inactive components from contrast - // requirements, but the audit flags them anyway. Inspect's - // Run button is disabled until a domain is typed, so the - // empty state reported a contrast failure that was never a - // real defect. Suppressing on the rule beats driving the UI - // to enable the control: typing raises the keyboard, which - // then follows the audit onto later screens and flags the - // system emoji picker's category buttons. - if issue.auditType.contains(.contrast), issue.element?.isEnabled == false { - return true - } - - let isEnforced = !enforcedAuditTypes.intersection(issue.auditType).isEmpty - let marker = isEnforced ? "FAIL" : "report" // Include the element so the burndown says *what* to fix, not // just that something is wrong. let element = issue.element.map { el -> String in let label = el.label.isEmpty ? el.identifier : el.label return label.isEmpty ? "\(el.elementType)" : "\"\(label)\"" } ?? "unknown element" + + // Characterised noise never fails, but is still logged with + // its reason — nothing disappears silently. + if let noise = noiseReason(for: issue) { + findings.append("[noise: \(noise)][\(name(for: issue.auditType))] \(issue.compactDescription) — \(element)") + return true + } + + let isEnforced = !reportOnly && !enforcedAuditTypes.intersection(issue.auditType).isEmpty + let marker = isEnforced ? "FAIL" : "report" findings.append("[\(marker)][\(name(for: issue.auditType))] \(issue.compactDescription) — \(element)") // true suppresses the finding, false reports it as a test failure. return !isEnforced @@ -149,6 +158,52 @@ enum AccessibilityAuditHarness { return true } + /// Classifies findings that are measurement artifacts, not app defects. + /// Each rule exists because it was proven, not assumed; the evidence is + /// recorded inline. A classified finding is logged with its reason and + /// never fails the build. + private static func noiseReason(for issue: XCUIAccessibilityAuditIssue) -> String? { + // WCAG 1.4.3 exempts inactive components from contrast requirements, + // but the audit flags them anyway. Proven on Inspect's Run button, + // disabled until a domain is typed. (Driving the UI to enable it was + // worse: the raised keyboard followed the audit onto later screens and + // flagged the emoji picker.) + if issue.auditType.contains(.contrast), issue.element?.isEnabled == false { + return "disabled control, WCAG 1.4.3 exempt" + } + + // "Nearly passed" is the audit's near-miss band, not a failure. The + // only occurrences are iOS-rendered Settings section headers, whose + // styling is the system's. + if issue.compactDescription.localizedCaseInsensitiveContains("nearly passed") { + return "near-miss, not a failure" + } + + // Placeholder text in text/search fields is reported clipped at ANY + // length — shortening "Search portfolio" to "Search" changed nothing — + // and the search field's hit region at accessibility sizes is the + // system's own control. Reading `elementType` here is safe; reading + // `frame` is not (it kills element attribution for the whole audit). + if let type = issue.element?.elementType, type == .searchField || type == .textField { + if issue.auditType.contains(.textClipped) || issue.auditType.contains(.hitRegion) { + return "system field placeholder/hit region, length-independent" + } + } + + // Unattributed clipped-text/dynamic-type findings. Bisection showed the + // audit loses attribution inside NavigationLink rows and + // children-ignored elements and then reports failures on content that + // is visually verified correct (and, for the one long-standing + // empty-watchlist phantom, renders nothing clipped at all). Named + // findings in these categories still enforce. + if issue.element == nil, + issue.auditType.contains(.textClipped) || issue.auditType.contains(.dynamicType) { + return "unattributed, audit artifact on ignored/link content" + } + + return nil + } + /// `XCUIAccessibilityAuditType` is an option set whose description is just a /// raw bitmask, which makes the burndown list unreadable. Resolve it against /// the named members rather than hard-coding bit positions, so this keeps |