diff options
| author | Christian Cleberg <[email protected]> | 2026-07-23 19:20:52 -0500 |
|---|---|---|
| committer | Christian Cleberg <[email protected]> | 2026-07-23 19:34:57 -0500 |
| commit | 7315ae3da604debfacfed9715a04dc91139a31e0 (patch) | |
| tree | 13febd9af1f44de01f05c9aef661b23cd97e2770 /DomainDigUITests/AccessibilityMetadataTests.swift | |
| parent | 655d11b59a273c14720952478ccebec20a9627bd (diff) | |
| download | domain-dig-7315ae3da604debfacfed9715a04dc91139a31e0.tar.gz domain-dig-7315ae3da604debfacfed9715a04dc91139a31e0.tar.bz2 domain-dig-7315ae3da604debfacfed9715a04dc91139a31e0.zip | |
test(a11y): Phase 6 verification — metadata assertions, middle-band sweep, 27.0 fix
Executes the Phase-6 manual runbook against the simulator, converting the
mechanically-checkable parts into permanent coverage and reporting the rest
honestly by tier.
- Fix an enforced `.dynamicType` failure surfaced by `audit-a11y.sh current`
on iOS 27.0: the Settings `Section("Services")` system header (app sets no
font; 18.6 floor and 26.x CI are clean). Narrow, proven `noiseReason`
carve-out scoped to dynamicType on the exact Settings header titles. Delta:
current FAIL -> SUCCEEDED, finding still prints as `[noise: …]`.
- AccessibilityMetadataTests: assert the icon-only control labels and the dense
Watchlist/Batch row label+value contracts (green on 18.6 and 27.0). These
were one-time manual VoiceOver checks; now they gate.
- Middle-band Dynamic Type sweep at AccessibilityL across the seeded screens.
Found no band-exclusive third bug (recorded), retained as regression
insurance for a band that historically shipped two.
- AccessibilityScreenshotTests: best-effort, non-gating capture utility used to
produce the cross-runtime Light/Dark/AXXXL screenshots (simctl appearance
does not propagate headlessly; driven through the in-app picker instead).
- Docs/ACCESSIBILITY_VERIFICATION_RESULTS.md: full pass/fail/not-executable
matrix + 15 screenshots. Notable positive result: Differentiate Without Color
IS verifiable via the global com.apple.Accessibility defaults domain.
Diffstat (limited to 'DomainDigUITests/AccessibilityMetadataTests.swift')
| -rw-r--r-- | DomainDigUITests/AccessibilityMetadataTests.swift | 160 |
1 files changed, 160 insertions, 0 deletions
diff --git a/DomainDigUITests/AccessibilityMetadataTests.swift b/DomainDigUITests/AccessibilityMetadataTests.swift new file mode 100644 index 0000000..3ee229d --- /dev/null +++ b/DomainDigUITests/AccessibilityMetadataTests.swift @@ -0,0 +1,160 @@ +import XCTest + +/// Mechanical assertions for the accessibility **metadata** the manual runbook +/// (issue #21, Phase 6) checks by hand: icon-only control labels, dense-row +/// label/value pairs, and toggle selected-state. +/// +/// `performAccessibilityAudit` (see `AccessibilityAuditTests`) validates +/// contrast, hit-region, clipping, and trait *correctness*, but it does not +/// assert that a specific control carries a specific spoken label — that a +/// refresh button says "Refresh all tracked domains" rather than "arrow +/// clockwise". Those strings were one-time manual VoiceOver checks; this file +/// converts the ones reachable without a live network lookup into permanent +/// regression coverage, so a relabel or a lost `.accessibilityValue` fails CI. +/// +/// What is deliberately **not** here, and why: +/// - Inspect toolbar Clear/Actions/Export, the bookmark (Save) toggle, and the +/// Timeline grouping control only appear after a completed lookup, which needs +/// the network — non-deterministic in CI. They stay in the manual pass. +/// - VoiceOver speech, the More Content rotor, and custom-content ordering are +/// not observable from XCUITest at all (the rotor is a VoiceOver feature, not +/// an element property). `.accessibilityCustomContent` does not surface as a +/// queryable value here, so the row assertions cover label + value only. +@MainActor +final class AccessibilityMetadataTests: XCTestCase { + override func setUp() { + continueAfterFailure = true + } + + // MARK: Icon-only control labels (runbook §3a) + + /// Every icon-only control reachable from the seeded launch state must + /// announce a purpose, never a raw SF Symbol name. + func testIconOnlyControlLabels() { + let app = AccessibilityAuditHarness.launch(seeded: true) + + // Dashboard refresh. + app.selectRootTab("Dashboard") + XCTAssertTrue( + app.buttons["Refresh all tracked domains"].waitForExistence(timeout: 5), + "Dashboard refresh lost its 'Refresh all tracked domains' label" + ) + + // Watchlist (Tracked Domains) add + filter. + openTrackedDomains(app) + XCTAssertTrue( + app.buttons["Add domain"].waitForExistence(timeout: 5), + "Watchlist add-domain lost its 'Add domain' label" + ) + XCTAssertTrue( + app.buttons["Filter and sort"].exists, + "Watchlist filter lost its 'Filter and sort' label" + ) + + // History's "Filter" menu (HistoryView.swift:109) is gated behind a + // non-empty history, which the seed fixtures do not populate, so it is + // not reachable here — it stays a verified-by-construction item in the + // results matrix rather than a flaky assertion. + } + + /// Workflows is Pro-gated; the seed harness forces Pro so its create button + /// is reachable. + func testWorkflowsCreateLabel() { + let app = AccessibilityAuditHarness.launch(seeded: true) + app.selectRootTab("Settings") + let workflows = app.buttons["Workflows"] + XCTAssertTrue(workflows.waitForExistence(timeout: 5), "Settings no longer offers Workflows") + workflows.tap() + XCTAssertTrue( + app.buttons["Create workflow"].waitForExistence(timeout: 5), + "Workflows create lost its 'Create workflow' label" + ) + } + + // MARK: Dense rows — label is the domain, value is the status (runbook §3d) + + /// The watchlist's dense rows collapse to a single VoiceOver element whose + /// label is the domain and whose value is availability. The badge title is + /// folded into that value (children: .ignore), which is the §3c "one word" + /// contract. + func testWatchlistRowLabelAndValue() { + let app = AccessibilityAuditHarness.launch(seeded: true) + openTrackedDomains(app) + + assertElement(in: app, label: "healthy.example", value: "Registered") + // The stress-length fixture with no known availability. + assertElement( + in: app, + label: "very-long-subdomain.observability.internal.staging.example", + value: "Unknown" + ) + } + + /// Batch result rows: domain as label, "<status>, <availability>" as value — + /// including the failed lookup, whose badge reads "Failed". + func testBatchRowLabelAndValue() { + let app = AccessibilityAuditHarness.launch(seeded: true) + app.selectRootTab("Inspect") + + assertElement(in: app, label: "broken.example", value: "Critical, Registered") + assertElement(in: app, label: "unreachable.example", value: "Failed, Unknown") + } + + // Toggle selected-state (runbook §3b) is intentionally not asserted here. + // The bookmark ("Save domain") and Pin ("Pin domain") toggles both live in + // the Inspect result's Domain section, reachable only after a completed live + // lookup — non-deterministic in CI. The watchlist's own pin is a swipe/menu + // action that carries no `.isSelected` trait, and the Audit checklist and + // picker rows need seeded audits / a multi-step gated flow the fixtures do + // not provide. These remain in the manual pass; the results matrix records + // each as verified-by-construction with its source line. + + // MARK: Helpers + + private func openTrackedDomains(_ app: XCUIApplication) { + app.selectRootTab("Settings") + let trackedDomains = app.buttons["Tracked Domains"] + XCTAssertTrue(trackedDomains.waitForExistence(timeout: 5), "Settings no longer offers Tracked Domains") + trackedDomains.tap() + } + + /// A `children: .ignore` row can surface as a button, cell, or other-element + /// depending on its container; match on label across the likely types. + private func firstElement(in app: XCUIApplication, label: String) -> XCUIElement? { + let predicate = NSPredicate(format: "label == %@", label) + for query in [app.buttons, app.cells, app.otherElements, app.staticTexts] { + let match = query.matching(predicate).firstMatch + if match.exists { return match } + } + return nil + } + + private func assertElement( + in app: XCUIApplication, + label: String, + value: String, + file: StaticString = #filePath, + line: UInt = #line + ) { + // Wait for the row to appear at all. + let predicate = NSPredicate(format: "label == %@", label) + let anyMatch = app.descendants(matching: .any).matching(predicate).firstMatch + XCTAssertTrue( + anyMatch.waitForExistence(timeout: 8), + "No accessibility element labelled \(label)", + file: file, + line: line + ) + guard let element = firstElement(in: app, label: label) else { + XCTFail("Element \(label) exists but not as a queryable button/cell/other", file: file, line: line) + return + } + XCTAssertEqual( + element.value as? String, + value, + "Element \(label) reported value \(String(describing: element.value)); expected \(value)", + file: file, + line: line + ) + } +} |
