diff options
Diffstat (limited to 'DomainDigUITests')
| -rw-r--r-- | DomainDigUITests/AccessibilityAuditHarness.swift | 32 | ||||
| -rw-r--r-- | DomainDigUITests/AccessibilityAuditTests.swift | 42 | ||||
| -rw-r--r-- | DomainDigUITests/AccessibilityMetadataTests.swift | 160 | ||||
| -rw-r--r-- | DomainDigUITests/AccessibilityScreenshotTests.swift | 105 |
4 files changed, 339 insertions, 0 deletions
diff --git a/DomainDigUITests/AccessibilityAuditHarness.swift b/DomainDigUITests/AccessibilityAuditHarness.swift index 64d0993..24f900e 100644 --- a/DomainDigUITests/AccessibilityAuditHarness.swift +++ b/DomainDigUITests/AccessibilityAuditHarness.swift @@ -201,9 +201,41 @@ enum AccessibilityAuditHarness { return "unattributed, audit artifact on ignored/link content" } + // iOS-27-only Settings `Section` header dynamicType finding. On iOS 27.0 + // (and only there) the audit reports "font sizes partially unsupported" + // against a Settings section header — the same system-rendered headers + // already carved out for the contrast near-miss above. Proof it is a + // system-chrome artifact, not an app defect: + // • Each is a plain `Section("Services")` etc. (ContentView.swift ~2768); + // the app sets no font, so the scaling is UIKit's `.footnote` header. + // • Version-specific: absent on the iOS 18.6 floor and on the 26.x + // runtime CI runs (both audit clean); it surfaces only under 27.0. + // ACCESSIBILITY.md's coverage table records the same asymmetry + // ("dynamicType finding 18.6 missed"). + // • Attribution is unstable run-to-run across the header set + // (Tier/Preferences/Services), exactly like the documented contrast + // flip — so it lands on whichever header the traversal reaches first. + // Overriding every Section header with a custom scaling `Text` to chase + // this was rejected for the contrast case (ACCESSIBILITY.md) for trading + // platform convention for nothing; the same holds here. Scoped to + // dynamicType on the exact Settings header titles so a real regression + // on app-controlled text still enforces. + if issue.auditType.contains(.dynamicType), + let label = issue.element?.label, + settingsSectionHeaders.contains(label) { + return "iOS-rendered Settings section header, app sets no font (27.0-only)" + } + return nil } + /// The Settings screen's `Section(_:)` header titles. UIKit renders these; + /// the app passes only a string literal. Used to scope the section-header + /// dynamicType carve-out narrowly (see `noiseReason`). + private static let settingsSectionHeaders: Set<String> = [ + "Tier", "Preferences", "Services", "Data", "About" + ] + /// `XCUIAccessibilityAuditType` is an option set whose description is just a /// raw bitmask, which makes the burndown list unreadable. Resolve it against /// the named members rather than hard-coding bit positions, so this keeps diff --git a/DomainDigUITests/AccessibilityAuditTests.swift b/DomainDigUITests/AccessibilityAuditTests.swift index 0a8f457..07bcbbf 100644 --- a/DomainDigUITests/AccessibilityAuditTests.swift +++ b/DomainDigUITests/AccessibilityAuditTests.swift @@ -112,6 +112,48 @@ final class AccessibilityAuditTests: XCTestCase { try XCTSkipUnless(audited, "Audit did not complete in time for seeded batch results") } + /// The seeded screens at an **intermediate** accessibility size. + /// + /// The default/`AccessibilityXXXL` pair brackets the range but skips the + /// middle band, and two production layout bugs lived exactly there — a + /// bordered button letter-wrapping vertically at a merely-large size, which + /// neither endpoint reproduced. `AccessibilityL` samples that band across the + /// dense seeded screens so a regression in it cannot slip between the two + /// existing test points. Reports rather than gates, matching the other + /// seeded audits. + func testSeededScreensAtIntermediateAccessibilitySize() throws { + let app = AccessibilityAuditHarness.launch( + contentSizeCategory: "UICTContentSizeCategoryAccessibilityL", + seeded: true + ) + + var unaudited: [String] = [] + + app.selectRootTab("Dashboard") + if try !AccessibilityAuditHarness.audit(app, screen: "seeded-dashboard-accessibilityL", test: self, reportOnly: true) { + unaudited.append("Dashboard") + } + + app.selectRootTab("Inspect") + if try !AccessibilityAuditHarness.audit(app, screen: "seeded-batch-accessibilityL", test: self, reportOnly: true) { + unaudited.append("Inspect batch") + } + + app.selectRootTab("Settings") + let trackedDomains = app.buttons["Tracked Domains"] + if trackedDomains.waitForExistence(timeout: 5) { + trackedDomains.tap() + if try !AccessibilityAuditHarness.audit(app, screen: "seeded-tracked-domains-accessibilityL", test: self, reportOnly: true) { + unaudited.append("Tracked Domains") + } + } + + try XCTSkipUnless( + unaudited.isEmpty, + "Audit did not complete in time for: \(unaudited.joined(separator: ", "))" + ) + } + /// The seeded screens again at the largest accessibility size — the case the /// deferred ViewThatFits work exists for. func testSeededScreensAtLargestAccessibilitySize() throws { diff --git a/DomainDigUITests/AccessibilityMetadataTests.swift b/DomainDigUITests/AccessibilityMetadataTests.swift new file mode 100644 index 0000000..3ee229d --- /dev/null +++ b/DomainDigUITests/AccessibilityMetadataTests.swift @@ -0,0 +1,160 @@ +import XCTest + +/// Mechanical assertions for the accessibility **metadata** the manual runbook +/// (issue #21, Phase 6) checks by hand: icon-only control labels, dense-row +/// label/value pairs, and toggle selected-state. +/// +/// `performAccessibilityAudit` (see `AccessibilityAuditTests`) validates +/// contrast, hit-region, clipping, and trait *correctness*, but it does not +/// assert that a specific control carries a specific spoken label — that a +/// refresh button says "Refresh all tracked domains" rather than "arrow +/// clockwise". Those strings were one-time manual VoiceOver checks; this file +/// converts the ones reachable without a live network lookup into permanent +/// regression coverage, so a relabel or a lost `.accessibilityValue` fails CI. +/// +/// What is deliberately **not** here, and why: +/// - Inspect toolbar Clear/Actions/Export, the bookmark (Save) toggle, and the +/// Timeline grouping control only appear after a completed lookup, which needs +/// the network — non-deterministic in CI. They stay in the manual pass. +/// - VoiceOver speech, the More Content rotor, and custom-content ordering are +/// not observable from XCUITest at all (the rotor is a VoiceOver feature, not +/// an element property). `.accessibilityCustomContent` does not surface as a +/// queryable value here, so the row assertions cover label + value only. +@MainActor +final class AccessibilityMetadataTests: XCTestCase { + override func setUp() { + continueAfterFailure = true + } + + // MARK: Icon-only control labels (runbook §3a) + + /// Every icon-only control reachable from the seeded launch state must + /// announce a purpose, never a raw SF Symbol name. + func testIconOnlyControlLabels() { + let app = AccessibilityAuditHarness.launch(seeded: true) + + // Dashboard refresh. + app.selectRootTab("Dashboard") + XCTAssertTrue( + app.buttons["Refresh all tracked domains"].waitForExistence(timeout: 5), + "Dashboard refresh lost its 'Refresh all tracked domains' label" + ) + + // Watchlist (Tracked Domains) add + filter. + openTrackedDomains(app) + XCTAssertTrue( + app.buttons["Add domain"].waitForExistence(timeout: 5), + "Watchlist add-domain lost its 'Add domain' label" + ) + XCTAssertTrue( + app.buttons["Filter and sort"].exists, + "Watchlist filter lost its 'Filter and sort' label" + ) + + // History's "Filter" menu (HistoryView.swift:109) is gated behind a + // non-empty history, which the seed fixtures do not populate, so it is + // not reachable here — it stays a verified-by-construction item in the + // results matrix rather than a flaky assertion. + } + + /// Workflows is Pro-gated; the seed harness forces Pro so its create button + /// is reachable. + func testWorkflowsCreateLabel() { + let app = AccessibilityAuditHarness.launch(seeded: true) + app.selectRootTab("Settings") + let workflows = app.buttons["Workflows"] + XCTAssertTrue(workflows.waitForExistence(timeout: 5), "Settings no longer offers Workflows") + workflows.tap() + XCTAssertTrue( + app.buttons["Create workflow"].waitForExistence(timeout: 5), + "Workflows create lost its 'Create workflow' label" + ) + } + + // MARK: Dense rows — label is the domain, value is the status (runbook §3d) + + /// The watchlist's dense rows collapse to a single VoiceOver element whose + /// label is the domain and whose value is availability. The badge title is + /// folded into that value (children: .ignore), which is the §3c "one word" + /// contract. + func testWatchlistRowLabelAndValue() { + let app = AccessibilityAuditHarness.launch(seeded: true) + openTrackedDomains(app) + + assertElement(in: app, label: "healthy.example", value: "Registered") + // The stress-length fixture with no known availability. + assertElement( + in: app, + label: "very-long-subdomain.observability.internal.staging.example", + value: "Unknown" + ) + } + + /// Batch result rows: domain as label, "<status>, <availability>" as value — + /// including the failed lookup, whose badge reads "Failed". + func testBatchRowLabelAndValue() { + let app = AccessibilityAuditHarness.launch(seeded: true) + app.selectRootTab("Inspect") + + assertElement(in: app, label: "broken.example", value: "Critical, Registered") + assertElement(in: app, label: "unreachable.example", value: "Failed, Unknown") + } + + // Toggle selected-state (runbook §3b) is intentionally not asserted here. + // The bookmark ("Save domain") and Pin ("Pin domain") toggles both live in + // the Inspect result's Domain section, reachable only after a completed live + // lookup — non-deterministic in CI. The watchlist's own pin is a swipe/menu + // action that carries no `.isSelected` trait, and the Audit checklist and + // picker rows need seeded audits / a multi-step gated flow the fixtures do + // not provide. These remain in the manual pass; the results matrix records + // each as verified-by-construction with its source line. + + // MARK: Helpers + + private func openTrackedDomains(_ app: XCUIApplication) { + app.selectRootTab("Settings") + let trackedDomains = app.buttons["Tracked Domains"] + XCTAssertTrue(trackedDomains.waitForExistence(timeout: 5), "Settings no longer offers Tracked Domains") + trackedDomains.tap() + } + + /// A `children: .ignore` row can surface as a button, cell, or other-element + /// depending on its container; match on label across the likely types. + private func firstElement(in app: XCUIApplication, label: String) -> XCUIElement? { + let predicate = NSPredicate(format: "label == %@", label) + for query in [app.buttons, app.cells, app.otherElements, app.staticTexts] { + let match = query.matching(predicate).firstMatch + if match.exists { return match } + } + return nil + } + + private func assertElement( + in app: XCUIApplication, + label: String, + value: String, + file: StaticString = #filePath, + line: UInt = #line + ) { + // Wait for the row to appear at all. + let predicate = NSPredicate(format: "label == %@", label) + let anyMatch = app.descendants(matching: .any).matching(predicate).firstMatch + XCTAssertTrue( + anyMatch.waitForExistence(timeout: 8), + "No accessibility element labelled \(label)", + file: file, + line: line + ) + guard let element = firstElement(in: app, label: label) else { + XCTFail("Element \(label) exists but not as a queryable button/cell/other", file: file, line: line) + return + } + XCTAssertEqual( + element.value as? String, + value, + "Element \(label) reported value \(String(describing: element.value)); expected \(value)", + file: file, + line: line + ) + } +} diff --git a/DomainDigUITests/AccessibilityScreenshotTests.swift b/DomainDigUITests/AccessibilityScreenshotTests.swift new file mode 100644 index 0000000..9cdafac --- /dev/null +++ b/DomainDigUITests/AccessibilityScreenshotTests.swift @@ -0,0 +1,105 @@ +import XCTest + +/// Captures the Phase-6 visual-pass evidence as result-bundle attachments: +/// the seeded dense screens in Light and Dark, and again at an accessibility +/// text size. Driven from XCUITest (not `simctl`) for two reasons proven during +/// this pass: the seed fixtures only populate under the test harness's launch, +/// and `xcrun simctl ui appearance` does not propagate to a headless-booted +/// simulator — so appearance is switched through the app's own Settings → +/// Display picker, exercising the real code path. +/// +/// **This is a capture utility, not a pass/fail test.** Every step is +/// best-effort and never asserts: a control it cannot reach on a given runtime +/// simply yields no screenshot, so adding it to the audit suite can never gate +/// CI. Run with an explicit `-resultBundlePath` and export the attachments. +@MainActor +final class AccessibilityScreenshotTests: XCTestCase { + override func setUp() { + continueAfterFailure = true + } + + /// Dashboard, Watchlist, and batch results in Light then Dark. + func testLightAndDarkScreens() { + let app = AccessibilityAuditHarness.launch(seeded: true) + + for appearance in ["Light", "Dark"] { + guard setAppearance(app, to: appearance) else { continue } + captureSeededScreens(app, suffix: appearance.lowercased()) + } + } + + /// The same seeded screens at the largest accessibility text size, where + /// reflow and clipping surface. Launched fresh with the content-size arg. + func testAccessibilityTextSizeScreens() { + let app = AccessibilityAuditHarness.launch( + contentSizeCategory: "UICTContentSizeCategoryAccessibilityXXXL", + seeded: true + ) + captureSeededScreens(app, suffix: "axxxl") + } + + // MARK: Capture (best-effort) + + private func captureSeededScreens(_ app: XCUIApplication, suffix: String) { + app.selectRootTab("Dashboard") + _ = app.buttons["Refresh all tracked domains"].waitForExistence(timeout: 5) + attach(app, name: "dashboard-\(suffix)") + + app.selectRootTab("Settings") + let trackedDomains = app.buttons["Tracked Domains"] + if trackedDomains.waitForExistence(timeout: 5) { + trackedDomains.tap() + _ = element(app, labelled: "healthy.example").waitForExistence(timeout: 5) + attach(app, name: "watchlist-\(suffix)") + } + + app.selectRootTab("Inspect") + _ = element(app, labelled: "broken.example").waitForExistence(timeout: 5) + attach(app, name: "batch-\(suffix)") + } + + private func element(_ app: XCUIApplication, labelled label: String) -> XCUIElement { + app.descendants(matching: .any) + .matching(NSPredicate(format: "label == %@", label)) + .firstMatch + } + + private func attach(_ app: XCUIApplication, name: String) { + let attachment = XCTAttachment(screenshot: app.screenshot()) + attachment.name = name + attachment.lifetime = .keepAlways + add(attachment) + } + + // MARK: Appearance (best-effort; returns whether it switched) + + /// Drives Settings → Display → Appearance to the given option. The `Picker` + /// renders as an inline `.menu` whose trigger is labelled + /// "Appearance, <current value>". Returns `false` (rather than failing) if + /// any step is unreachable on this runtime. + private func setAppearance(_ app: XCUIApplication, to option: String) -> Bool { + // A prior capture may have left the Settings tab on a pushed view + // (Tracked Domains). Re-selecting the active tab pops it back to root. + app.selectRootTab("Settings") + let display = app.buttons["Display"] + if !display.waitForExistence(timeout: 2) { + app.selectRootTab("Settings") + } + guard display.waitForExistence(timeout: 5) else { return false } + display.tap() + + let trigger = app.buttons + .matching(NSPredicate(format: "label BEGINSWITH %@", "Appearance")) + .firstMatch + guard trigger.waitForExistence(timeout: 5) else { return false } + trigger.tap() + + // The popped menu exposes System / Light / Dark as buttons (or menu + // items on some runtimes). + let asButton = app.buttons[option] + let choice = asButton.waitForExistence(timeout: 3) ? asButton : app.menuItems[option] + guard choice.waitForExistence(timeout: 3) else { return false } + choice.tap() + return true + } +} |