1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
import XCTest
/// Shared plumbing for the accessibility audit suite.
///
/// `performAccessibilityAudit` checks contrast, hit-region size, clipped text at
/// large Dynamic Type, element descriptions, and trait correctness — the same
/// categories the accessibility pass in issue #21 works through.
///
/// **The suite reports by default and fails only for enforced categories.** The
/// audit surfaces violations that exist today, so failing on everything would
/// block unrelated PRs until the whole pass lands. `enforcedAuditTypes` below is
/// the ratchet: widen it as each phase of #21 clears a category.
///
/// Two alternatives were tried and rejected:
///
/// - *A per-screen baseline count.* Audit coverage is not nested across OS
/// versions — the same screen legitimately yields different counts on the
/// floor simulator and the current one, so no single committed number is
/// correct for both.
/// - *An environment variable.* Neither a plain `xcodebuild` env var nor a
/// `TEST_RUNNER_`-prefixed build setting reaches this process, so the toggle
/// silently did nothing. A committed constant also makes "when did contrast
/// become enforced?" answerable with `git blame` instead of CI tribal
/// knowledge.
@MainActor
enum AccessibilityAuditHarness {
/// Launch argument that lifts feature gating so Pro-only screens are
/// reachable. `PurchaseService` honours this in `DEBUG` builds only.
private static let forceProPlusArgument = "DOMAIN_DIG_FORCE_PRO_PLUS"
/// Audit categories that fail the build. Everything else is reported only.
///
/// Empty until the accessibility pass starts landing. Suggested ratchet,
/// following the phases in issue #21:
///
/// - after phase 2 (semantic colors + light mode): `.contrast`
/// - after phase 3 (Dynamic Type + reflow): `.textClipped`, `.dynamicType`,
/// `.hitRegion`
/// - after phase 4 (VoiceOver): `.elementDetection`,
/// `.sufficientElementDescription`, `.trait`
static let enforcedAuditTypes: XCUIAccessibilityAuditType = []
/// How many times to retry an audit that misses its internal deadline.
private static let auditAttempts = 3
/// Launches the app with feature gating lifted, optionally at a specific
/// content size category.
static func launch(contentSizeCategory: String? = nil) -> XCUIApplication {
let app = XCUIApplication()
app.launchArguments = [forceProPlusArgument]
if let contentSizeCategory {
app.launchArguments += ["-UIPreferredContentSizeCategoryName", contentSizeCategory]
}
app.launch()
return app
}
/// Runs a full audit and records every finding against the test.
///
/// Findings are logged and attached to the result bundle so a CI run
/// produces the burndown list as an artifact rather than only a pass/fail.
///
/// Returns `false` if the audit could not complete, leaving the screen
/// unaudited. Callers turn that into an `XCTSkip` — reporting a pass would
/// claim coverage that did not happen.
@discardableResult
static func audit(
_ app: XCUIApplication,
screen: String,
test: XCTestCase
) throws -> Bool {
var findings: [String] = []
var timeout: Error?
// The audit traverses the whole element tree and has its own internal
// deadline, which slower CI runners miss on the denser screens. That is a
// tooling timeout, not an app defect, so retry before giving up.
//
// Only the timeout is retried. If a category is enforced and the audit
// reports findings before timing out, those failures are already recorded
// and a retry would duplicate them — accepted, because the alternative is
// losing the run to an infrastructure hiccup.
for attempt in 1...auditAttempts {
findings.removeAll()
timeout = nil
do {
try app.performAccessibilityAudit { issue in
// WCAG 1.4.3 exempts inactive components from contrast
// requirements, but the audit flags them anyway. Inspect's
// Run button is disabled until a domain is typed, so the
// empty state reported a contrast failure that was never a
// real defect. Suppressing on the rule beats driving the UI
// to enable the control: typing raises the keyboard, which
// then follows the audit onto later screens and flags the
// system emoji picker's category buttons.
if issue.auditType.contains(.contrast), issue.element?.isEnabled == false {
return true
}
let isEnforced = !enforcedAuditTypes.intersection(issue.auditType).isEmpty
let marker = isEnforced ? "FAIL" : "report"
// Include the element so the burndown says *what* to fix, not
// just that something is wrong.
let element = issue.element.map { el -> String in
let label = el.label.isEmpty ? el.identifier : el.label
return label.isEmpty ? "\(el.elementType)" : "\"\(label)\""
} ?? "unknown element"
findings.append("[\(marker)][\(name(for: issue.auditType))] \(issue.compactDescription) — \(element)")
// true suppresses the finding, false reports it as a test failure.
return !isEnforced
}
break
} catch let error as NSError where error.isAccessibilityAuditTimeout {
timeout = error
print("\(screen): audit timed out (attempt \(attempt) of \(auditAttempts))")
}
}
if timeout != nil {
let message = "\(screen): audit did not complete in time after \(auditAttempts) attempts — screen NOT audited"
print(message)
let attachment = XCTAttachment(string: message)
attachment.name = "a11y-audit-\(screen)-timeout"
attachment.lifetime = .keepAlways
test.add(attachment)
return false
}
let summary = findings.isEmpty
? "\(screen): no accessibility findings"
: "\(screen): \(findings.count) finding(s)\n" + findings.sorted().map { " • \($0)" }.joined(separator: "\n")
print(summary)
let attachment = XCTAttachment(string: summary)
attachment.name = "a11y-audit-\(screen)"
attachment.lifetime = .keepAlways
test.add(attachment)
return true
}
/// `XCUIAccessibilityAuditType` is an option set whose description is just a
/// raw bitmask, which makes the burndown list unreadable. Resolve it against
/// the named members rather than hard-coding bit positions, so this keeps
/// working if Apple adds audit types.
private static func name(for type: XCUIAccessibilityAuditType) -> String {
let known: [(XCUIAccessibilityAuditType, String)] = [
(.contrast, "contrast"),
(.elementDetection, "elementDetection"),
(.hitRegion, "hitRegion"),
(.sufficientElementDescription, "sufficientElementDescription"),
(.dynamicType, "dynamicType"),
(.textClipped, "textClipped"),
(.trait, "trait")
]
let matched = known.filter { type.contains($0.0) }.map(\.1)
return matched.isEmpty ? "unknown(\(type.rawValue))" : matched.joined(separator: "+")
}
}
private extension NSError {
/// `Audit failed to complete in time` — the audit's own deadline, raised by
/// XCTest rather than by anything wrong with the app.
var isAccessibilityAuditTimeout: Bool {
domain == "com.apple.xcode.xctest.accessibilityAudit" && code == -56
}
}
extension XCUIApplication {
/// Taps a root tab by its visible label.
///
/// Falls back to a plain button query because the tab bar is only present in
/// the compact size class — in regular width `RootTabView` renders a
/// `NavigationSplitView` sidebar instead.
@MainActor
func selectRootTab(_ name: String) {
let tabButton = tabBars.buttons[name]
let element = tabButton.waitForExistence(timeout: 5) ? tabButton : buttons[name]
XCTAssertTrue(
element.waitForExistence(timeout: 5),
"Could not find a way to reach the \(name) screen"
)
element.tap()
}
}
|
