diff options
| -rw-r--r-- | Docs/ACCESSIBILITY.md | 53 | ||||
| -rw-r--r-- | DomainDigUITests/AccessibilityAuditHarness.swift | 105 | ||||
| -rw-r--r-- | DomainDigUITests/AccessibilityAuditTests.swift | 12 |
3 files changed, 118 insertions, 52 deletions
diff --git a/Docs/ACCESSIBILITY.md b/Docs/ACCESSIBILITY.md index b35f02c..3f8b3ba 100644 --- a/Docs/ACCESSIBILITY.md +++ b/Docs/ACCESSIBILITY.md @@ -110,27 +110,38 @@ Users override it under Settings → Display. Dark mode reports 18 findings and light mode 21; the three extra are the section headers above. Everything the app actually controls passes in both schemes. -## Findings are reported, not enforced - -The audit surfaces violations that exist today, so failing on all of them would -block every unrelated change until the whole pass lands. Instead, findings are -logged and attached to the result bundle tagged `[report]` or `[FAIL]`. - -Enforcement is the committed constant -`AccessibilityAuditHarness.enforcedAuditTypes`. Widen it as each phase clears a -category: - -| After phase | Enforce | -| --- | --- | -| 2 — semantic colors + light mode | `.contrast` | -| 3 — Dynamic Type + reflow | `.textClipped`, `.dynamicType`, `.hitRegion` | -| 4 — VoiceOver | `.elementDetection`, `.sufficientElementDescription`, `.trait` | - -A constant rather than a CI setting, for two reasons. Environment variables do -not work: neither a plain `xcodebuild` env var nor a `TEST_RUNNER_`-prefixed -build setting reaches the UI test process, so the toggle silently did nothing. -And a committed value makes "when did contrast become enforced?" answerable with -`git blame` instead of CI tribal knowledge. +## Enforcement — the ratchet is engaged + +With phases 1–5 landed, `AccessibilityAuditHarness.enforcedAuditTypes` enforces +**`.textClipped`, `.dynamicType`, `.hitRegion`, `.elementDetection`, +`.sufficientElementDescription`, `.trait`** on the empty-state test suite. A +named finding in any of these fails CI — regressions in five phases of work are +now gated, not merely reported. + +Three deliberate carve-outs, each with its evidence: + +1. **`.contrast` stays report-only.** The two long-standing Settings findings + are rows scrolled under the translucent tab bar; their attribution flips + between a row name and nil run-to-run, so no suppression is narrow enough to + keep CI stable. The centralised palette in `Shared/Colors.xcassets` is the + actual guard against contrast regressions. +2. **The seeded tests run `reportOnly`.** Bisecting the row/badge accessibility + modifiers showed the audit degrades on `children: .ignore` content — the + *correct* VoiceOver treatment for dense rows — emitting unattributed + contrast/dynamicType failures on rows that measure 6–7:1 and render + correctly. Their burndown still prints; it just doesn't gate. +3. **Characterised noise is suppressed narrowly and always logged** with a + `[noise: reason]` marker — disabled controls (WCAG 1.4.3 exempt), "nearly + passed" near-misses, system field placeholders (clipped at any length — + proven by shortening them to no effect), and unattributed + clipped/dynamic-type artifacts. Nothing disappears silently; see + `noiseReason(for:)` for each rule's provenance. + +Enforcement is a committed constant rather than a CI setting, for two reasons. +Environment variables do not work: neither a plain `xcodebuild` env var nor a +`TEST_RUNNER_`-prefixed build setting reaches the UI test process, so the toggle +silently did nothing. And a committed value makes "when did clipping become +enforced?" answerable with `git blame` instead of CI tribal knowledge. ## Why coverage is split between local and CI diff --git a/DomainDigUITests/AccessibilityAuditHarness.swift b/DomainDigUITests/AccessibilityAuditHarness.swift index e025dcd..64d0993 100644 --- a/DomainDigUITests/AccessibilityAuditHarness.swift +++ b/DomainDigUITests/AccessibilityAuditHarness.swift @@ -28,17 +28,23 @@ enum AccessibilityAuditHarness { /// reachable. `PurchaseService` honours this in `DEBUG` builds only. private static let forceProPlusArgument = "DOMAIN_DIG_FORCE_PRO_PLUS" - /// Audit categories that fail the build. Everything else is reported only. + /// Audit categories that fail the build on the empty-state suite. A named + /// finding in any of these is a regression in the phase 1–5 work. /// - /// Empty until the accessibility pass starts landing. Suggested ratchet, - /// following the phases in issue #21: - /// - /// - after phase 2 (semantic colors + light mode): `.contrast` - /// - after phase 3 (Dynamic Type + reflow): `.textClipped`, `.dynamicType`, - /// `.hitRegion` - /// - after phase 4 (VoiceOver): `.elementDetection`, - /// `.sufficientElementDescription`, `.trait` - static let enforcedAuditTypes: XCUIAccessibilityAuditType = [] + /// `.contrast` is deliberately absent: the two long-standing Settings + /// findings come from rows scrolled under the translucent tab bar, and their + /// attribution flips between a row name and nil run-to-run, so there is no + /// suppression narrow enough to keep CI stable. Contrast stays report-only, + /// with the palette centralised in `Shared/Colors.xcassets` as the actual + /// guard. + static let enforcedAuditTypes: XCUIAccessibilityAuditType = [ + .textClipped, + .dynamicType, + .hitRegion, + .elementDetection, + .sufficientElementDescription, + .trait + ] /// How many times to retry an audit that misses its internal deadline. private static let auditAttempts = 3 @@ -72,11 +78,18 @@ enum AccessibilityAuditHarness { /// Returns `false` if the audit could not complete, leaving the screen /// unaudited. Callers turn that into an `XCTSkip` — reporting a pass would /// claim coverage that did not happen. + /// `reportOnly` disables enforcement for this call. Used by the seeded + /// tests: bisection showed the audit degrades on `children: .ignore` + /// content — the correct VoiceOver treatment for dense rows — reporting + /// unattributed contrast/dynamicType failures on rows that measure 6–7:1 + /// and render correctly. Until that behaves, the seeded screens report + /// their burndown without gating CI. @discardableResult static func audit( _ app: XCUIApplication, screen: String, - test: XCTestCase + test: XCTestCase, + reportOnly: Bool = false ) throws -> Bool { var findings: [String] = [] var timeout: Error? @@ -94,26 +107,22 @@ enum AccessibilityAuditHarness { timeout = nil do { try app.performAccessibilityAudit { issue in - // WCAG 1.4.3 exempts inactive components from contrast - // requirements, but the audit flags them anyway. Inspect's - // Run button is disabled until a domain is typed, so the - // empty state reported a contrast failure that was never a - // real defect. Suppressing on the rule beats driving the UI - // to enable the control: typing raises the keyboard, which - // then follows the audit onto later screens and flags the - // system emoji picker's category buttons. - if issue.auditType.contains(.contrast), issue.element?.isEnabled == false { - return true - } - - let isEnforced = !enforcedAuditTypes.intersection(issue.auditType).isEmpty - let marker = isEnforced ? "FAIL" : "report" // Include the element so the burndown says *what* to fix, not // just that something is wrong. let element = issue.element.map { el -> String in let label = el.label.isEmpty ? el.identifier : el.label return label.isEmpty ? "\(el.elementType)" : "\"\(label)\"" } ?? "unknown element" + + // Characterised noise never fails, but is still logged with + // its reason — nothing disappears silently. + if let noise = noiseReason(for: issue) { + findings.append("[noise: \(noise)][\(name(for: issue.auditType))] \(issue.compactDescription) — \(element)") + return true + } + + let isEnforced = !reportOnly && !enforcedAuditTypes.intersection(issue.auditType).isEmpty + let marker = isEnforced ? "FAIL" : "report" findings.append("[\(marker)][\(name(for: issue.auditType))] \(issue.compactDescription) — \(element)") // true suppresses the finding, false reports it as a test failure. return !isEnforced @@ -149,6 +158,52 @@ enum AccessibilityAuditHarness { return true } + /// Classifies findings that are measurement artifacts, not app defects. + /// Each rule exists because it was proven, not assumed; the evidence is + /// recorded inline. A classified finding is logged with its reason and + /// never fails the build. + private static func noiseReason(for issue: XCUIAccessibilityAuditIssue) -> String? { + // WCAG 1.4.3 exempts inactive components from contrast requirements, + // but the audit flags them anyway. Proven on Inspect's Run button, + // disabled until a domain is typed. (Driving the UI to enable it was + // worse: the raised keyboard followed the audit onto later screens and + // flagged the emoji picker.) + if issue.auditType.contains(.contrast), issue.element?.isEnabled == false { + return "disabled control, WCAG 1.4.3 exempt" + } + + // "Nearly passed" is the audit's near-miss band, not a failure. The + // only occurrences are iOS-rendered Settings section headers, whose + // styling is the system's. + if issue.compactDescription.localizedCaseInsensitiveContains("nearly passed") { + return "near-miss, not a failure" + } + + // Placeholder text in text/search fields is reported clipped at ANY + // length — shortening "Search portfolio" to "Search" changed nothing — + // and the search field's hit region at accessibility sizes is the + // system's own control. Reading `elementType` here is safe; reading + // `frame` is not (it kills element attribution for the whole audit). + if let type = issue.element?.elementType, type == .searchField || type == .textField { + if issue.auditType.contains(.textClipped) || issue.auditType.contains(.hitRegion) { + return "system field placeholder/hit region, length-independent" + } + } + + // Unattributed clipped-text/dynamic-type findings. Bisection showed the + // audit loses attribution inside NavigationLink rows and + // children-ignored elements and then reports failures on content that + // is visually verified correct (and, for the one long-standing + // empty-watchlist phantom, renders nothing clipped at all). Named + // findings in these categories still enforce. + if issue.element == nil, + issue.auditType.contains(.textClipped) || issue.auditType.contains(.dynamicType) { + return "unattributed, audit artifact on ignored/link content" + } + + return nil + } + /// `XCUIAccessibilityAuditType` is an option set whose description is just a /// raw bitmask, which makes the burndown list unreadable. Resolve it against /// the named members rather than hard-coding bit positions, so this keeps diff --git a/DomainDigUITests/AccessibilityAuditTests.swift b/DomainDigUITests/AccessibilityAuditTests.swift index 0863d32..0a8f457 100644 --- a/DomainDigUITests/AccessibilityAuditTests.swift +++ b/DomainDigUITests/AccessibilityAuditTests.swift @@ -89,7 +89,7 @@ final class AccessibilityAuditTests: XCTestCase { func testSeededDashboard() throws { let app = AccessibilityAuditHarness.launch(seeded: true) app.selectRootTab("Dashboard") - let audited = try AccessibilityAuditHarness.audit(app, screen: "seeded-dashboard", test: self) + let audited = try AccessibilityAuditHarness.audit(app, screen: "seeded-dashboard", test: self, reportOnly: true) try XCTSkipUnless(audited, "Audit did not complete in time for seeded Dashboard") } @@ -100,7 +100,7 @@ final class AccessibilityAuditTests: XCTestCase { let trackedDomains = app.buttons["Tracked Domains"] XCTAssertTrue(trackedDomains.waitForExistence(timeout: 5)) trackedDomains.tap() - let audited = try AccessibilityAuditHarness.audit(app, screen: "seeded-tracked-domains", test: self) + let audited = try AccessibilityAuditHarness.audit(app, screen: "seeded-tracked-domains", test: self, reportOnly: true) try XCTSkipUnless(audited, "Audit did not complete in time for seeded Tracked Domains") } @@ -108,7 +108,7 @@ final class AccessibilityAuditTests: XCTestCase { func testSeededBatchResults() throws { let app = AccessibilityAuditHarness.launch(seeded: true) app.selectRootTab("Inspect") - let audited = try AccessibilityAuditHarness.audit(app, screen: "seeded-batch", test: self) + let audited = try AccessibilityAuditHarness.audit(app, screen: "seeded-batch", test: self, reportOnly: true) try XCTSkipUnless(audited, "Audit did not complete in time for seeded batch results") } @@ -123,12 +123,12 @@ final class AccessibilityAuditTests: XCTestCase { var unaudited: [String] = [] app.selectRootTab("Dashboard") - if try !AccessibilityAuditHarness.audit(app, screen: "seeded-dashboard-accessibilityXXXL", test: self) { + if try !AccessibilityAuditHarness.audit(app, screen: "seeded-dashboard-accessibilityXXXL", test: self, reportOnly: true) { unaudited.append("Dashboard") } app.selectRootTab("Inspect") - if try !AccessibilityAuditHarness.audit(app, screen: "seeded-batch-accessibilityXXXL", test: self) { + if try !AccessibilityAuditHarness.audit(app, screen: "seeded-batch-accessibilityXXXL", test: self, reportOnly: true) { unaudited.append("Inspect batch") } @@ -136,7 +136,7 @@ final class AccessibilityAuditTests: XCTestCase { let trackedDomains = app.buttons["Tracked Domains"] if trackedDomains.waitForExistence(timeout: 5) { trackedDomains.tap() - if try !AccessibilityAuditHarness.audit(app, screen: "seeded-tracked-domains-accessibilityXXXL", test: self) { + if try !AccessibilityAuditHarness.audit(app, screen: "seeded-tracked-domains-accessibilityXXXL", test: self, reportOnly: true) { unaudited.append("Tracked Domains") } } |