diff options
| -rw-r--r-- | .github/workflows/build.yml | 41 | ||||
| -rw-r--r-- | DomainDigUITests/AccessibilityAuditHarness.swift | 61 | ||||
| -rw-r--r-- | DomainDigUITests/AccessibilityAuditTests.swift | 51 |
3 files changed, 114 insertions, 39 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a6a2113..e0c2eee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,18 +20,28 @@ name: Build # TEST_RUNNER_-prefixed build setting reaches the UI test process.) # # The matrix runs two simulators because audit coverage is NOT nested — each -# runtime reports findings the other misses, in both directions. Measured on the -# Tracked Domains screen, iOS 18.6 reported 2 issues and iOS 27.0 reported 6 -# (including contrast and element-detection issues 18.6 never raised); at -# accessibility text sizes the Dashboard produced a hit-region finding on 18.6 -# that 27.0 did not. Testing only the newest image would leave the oldest -# supported OS unchecked; testing only the floor would miss newer audit checks. +# runtime reports findings the other misses, in both directions. Measured +# locally on the Tracked Domains screen, iOS 18.6 reported 2 issues and iOS 27.0 +# reported 6 (including contrast and element-detection issues 18.6 never +# raised); at accessibility text sizes the Dashboard produced a hit-region +# finding on 18.6 that 27.0 did not. # -# Runtimes are resolved dynamically rather than pinned: the deployment target is -# 17.6, but no 17.6 simulator runtime ships, so "floor" means the oldest -# available runtime at or above the deployment target (18.6 at time of writing). -# The previous selector took the first iPhone from any runtime, which could pick -# a simulator BELOW the deployment target, where the app cannot install. +# CAVEAT — this image cannot test the real support floor. The app's deployment +# target is 17.6, but the macos-26 runner ships only iOS 26.x simulator +# runtimes, so "floor" resolves to ~26.2 here rather than an 18.x image. CI +# therefore compares two 26.x runtimes; genuine oldest-supported-OS coverage has +# to come from a local run or a self-hosted runner with older runtimes +# installed. The "Select simulator" step emits a warning annotation when the +# resolved floor sits well above the deployment target, so this gap stays +# visible instead of being silently assumed away. +# +# Installing an older runtime in CI (xcodebuild -downloadPlatform iOS +# -buildVersion 18.6) is possible but costs several GB and minutes per job; not +# done by default. +# +# Runtimes are resolved dynamically rather than pinned. The previous selector +# took the first iPhone from any runtime, which could pick a simulator BELOW the +# deployment target, where the app cannot install. # # pull_request only, plus manual dispatch. GitHub builds the merge result (PR # merged into main), so a green PR validates exactly what will land on main. @@ -115,6 +125,15 @@ jobs: echo "udid=$(echo "$selected" | jq -r .udid)" >> "$GITHUB_OUTPUT" echo "label=$label" >> "$GITHUB_OUTPUT" + # Surface the floor-coverage gap rather than letting the matrix imply + # coverage it does not have. One major version of slack is tolerated. + if [ "${{ matrix.tier }}" = "floor" ]; then + rank=$(echo "$selected" | jq -r .rank) + if [ "$rank" -ge $(( (DEPLOYMENT_TARGET_MAJOR + 1) * 1000 )) ]; then + echo "::warning::Floor tier resolved to $label, well above the ${DEPLOYMENT_TARGET_MAJOR}.${DEPLOYMENT_TARGET_MINOR} deployment target. This image has no older runtime, so the oldest supported OS is NOT covered by this run." + fi + fi + - name: Test on ${{ steps.sim.outputs.label }} run: | set -o pipefail diff --git a/DomainDigUITests/AccessibilityAuditHarness.swift b/DomainDigUITests/AccessibilityAuditHarness.swift index 80257e1..9767d95 100644 --- a/DomainDigUITests/AccessibilityAuditHarness.swift +++ b/DomainDigUITests/AccessibilityAuditHarness.swift @@ -40,6 +40,9 @@ enum AccessibilityAuditHarness { /// `.sufficientElementDescription`, `.trait` static let enforcedAuditTypes: XCUIAccessibilityAuditType = [] + /// How many times to retry an audit that misses its internal deadline. + private static let auditAttempts = 3 + /// Launches the app with feature gating lifted, optionally at a specific /// content size category. static func launch(contentSizeCategory: String? = nil) -> XCUIApplication { @@ -56,19 +59,53 @@ enum AccessibilityAuditHarness { /// /// Findings are logged and attached to the result bundle so a CI run /// produces the burndown list as an artifact rather than only a pass/fail. + /// + /// Returns `false` if the audit could not complete, leaving the screen + /// unaudited. Callers turn that into an `XCTSkip` — reporting a pass would + /// claim coverage that did not happen. + @discardableResult static func audit( _ app: XCUIApplication, screen: String, test: XCTestCase - ) throws { + ) throws -> Bool { var findings: [String] = [] + var timeout: Error? + + // The audit traverses the whole element tree and has its own internal + // deadline, which slower CI runners miss on the denser screens. That is a + // tooling timeout, not an app defect, so retry before giving up. + // + // Only the timeout is retried. If a category is enforced and the audit + // reports findings before timing out, those failures are already recorded + // and a retry would duplicate them — accepted, because the alternative is + // losing the run to an infrastructure hiccup. + for attempt in 1...auditAttempts { + findings.removeAll() + timeout = nil + do { + try app.performAccessibilityAudit { issue in + let isEnforced = !enforcedAuditTypes.intersection(issue.auditType).isEmpty + let marker = isEnforced ? "FAIL" : "report" + findings.append("[\(marker)][\(name(for: issue.auditType))] \(issue.compactDescription)") + // true suppresses the finding, false reports it as a test failure. + return !isEnforced + } + break + } catch let error as NSError where error.isAccessibilityAuditTimeout { + timeout = error + print("\(screen): audit timed out (attempt \(attempt) of \(auditAttempts))") + } + } - try app.performAccessibilityAudit { issue in - let isEnforced = !enforcedAuditTypes.intersection(issue.auditType).isEmpty - let marker = isEnforced ? "FAIL" : "report" - findings.append("[\(marker)][\(name(for: issue.auditType))] \(issue.compactDescription)") - // true suppresses the finding, false reports it as a test failure. - return !isEnforced + if timeout != nil { + let message = "\(screen): audit did not complete in time after \(auditAttempts) attempts — screen NOT audited" + print(message) + let attachment = XCTAttachment(string: message) + attachment.name = "a11y-audit-\(screen)-timeout" + attachment.lifetime = .keepAlways + test.add(attachment) + return false } let summary = findings.isEmpty @@ -81,6 +118,8 @@ enum AccessibilityAuditHarness { attachment.name = "a11y-audit-\(screen)" attachment.lifetime = .keepAlways test.add(attachment) + + return true } /// `XCUIAccessibilityAuditType` is an option set whose description is just a @@ -102,6 +141,14 @@ enum AccessibilityAuditHarness { } } +private extension NSError { + /// `Audit failed to complete in time` — the audit's own deadline, raised by + /// XCTest rather than by anything wrong with the app. + var isAccessibilityAuditTimeout: Bool { + domain == "com.apple.xcode.xctest.accessibilityAudit" && code == -56 + } +} + extension XCUIApplication { /// Taps a root tab by its visible label. /// diff --git a/DomainDigUITests/AccessibilityAuditTests.swift b/DomainDigUITests/AccessibilityAuditTests.swift index 63b3d02..6a90bac 100644 --- a/DomainDigUITests/AccessibilityAuditTests.swift +++ b/DomainDigUITests/AccessibilityAuditTests.swift @@ -17,33 +17,23 @@ final class AccessibilityAuditTests: XCTestCase { // MARK: Per-screen audits func testInspectScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Inspect") - try AccessibilityAuditHarness.audit(app, screen: "inspect", test: self) + try auditRootTab("Inspect") } func testDashboardScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Dashboard") - try AccessibilityAuditHarness.audit(app, screen: "dashboard", test: self) + try auditRootTab("Dashboard") } func testAuditScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Audit") - try AccessibilityAuditHarness.audit(app, screen: "audit", test: self) + try auditRootTab("Audit") } func testHistoryScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("History") - try AccessibilityAuditHarness.audit(app, screen: "history", test: self) + try auditRootTab("History") } func testSettingsScreen() throws { - let app = AccessibilityAuditHarness.launch() - app.selectRootTab("Settings") - try AccessibilityAuditHarness.audit(app, screen: "settings", test: self) + try auditRootTab("Settings") } func testTrackedDomainsScreen() throws { @@ -57,7 +47,8 @@ final class AccessibilityAuditTests: XCTestCase { ) trackedDomains.tap() - try AccessibilityAuditHarness.audit(app, screen: "tracked-domains", test: self) + let audited = try AccessibilityAuditHarness.audit(app, screen: "tracked-domains", test: self) + try XCTSkipUnless(audited, "Tracked Domains audit did not complete in time") } // MARK: Dynamic Type @@ -67,18 +58,36 @@ final class AccessibilityAuditTests: XCTestCase { /// This is where clipped text and fixed-height containers surface — the /// `.accessibility5`-class failures that the fixed geometry in /// `AppDensityMetrics` is expected to produce until phase 3 of #21 lands. + /// + /// Every screen is attempted even if an earlier one times out, so one slow + /// screen cannot silently drop the rest; the skip is reported at the end. func testAllScreensAtLargestAccessibilitySize() throws { let app = AccessibilityAuditHarness.launch( contentSizeCategory: "UICTContentSizeCategoryAccessibilityXXXL" ) + var unaudited: [String] = [] + for tab in ["Inspect", "Dashboard", "Audit", "History", "Settings"] { app.selectRootTab(tab) - try AccessibilityAuditHarness.audit( - app, - screen: "\(tab.lowercased())-accessibilityXXXL", - test: self - ) + let screen = "\(tab.lowercased())-accessibilityXXXL" + if try !AccessibilityAuditHarness.audit(app, screen: screen, test: self) { + unaudited.append(tab) + } } + + try XCTSkipUnless( + unaudited.isEmpty, + "Audit did not complete in time for: \(unaudited.joined(separator: ", "))" + ) + } + + // MARK: Helpers + + private func auditRootTab(_ tab: String) throws { + let app = AccessibilityAuditHarness.launch() + app.selectRootTab(tab) + let audited = try AccessibilityAuditHarness.audit(app, screen: tab.lowercased(), test: self) + try XCTSkipUnless(audited, "Audit did not complete in time for \(tab)") } } |