From b383ce36c80b1570d513606e5c304576c3da3232 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Mon, 19 Oct 2026 11:26:41 -0700 Subject: [PATCH] test(ios): drive menus, alerts and list rows from SNOWBOUND_SCRIPT activate: now picks a menu's item, a list's row and an alert's button, reaches a button within a row as `Row/Button`, prefers the frontmost of two matches, and keeps running after the page that started it goes. Its search no longer doubles at each level of the view tree, which took minutes on an iPad. Assisted-by: claude-opus-5.5 --- apps/ios/Snowbound/Script.swift | 66 +++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/apps/ios/Snowbound/Script.swift b/apps/ios/Snowbound/Script.swift index 803ce0d9ab7c405108e458dff56422b31959a931..3149e4aff85e9b191962966931d20686a547934a 100644 --- a/apps/ios/Snowbound/Script.swift +++ b/apps/ios/Snowbound/Script.swift @@ -20,8 +20,8 @@ extension CanvasView { /// Documents/tree.txt, and `shot:a` the window to Documents/a.png, as a device has no /// screenshot command; `snap:a` only logs, for a simulator screenshot with the keyboard; /// `activate:Label` activates the element of any window so named, as VoiceOver would, - /// opening a button's menu, and `audit` logs whether the format bar and tag grid name every - /// button. + /// opening a button's menu or picking a menu's item or a list's row (`Row/Button` for a + /// button in a row), and `audit` logs whether the format bar and tag grid name every button. func runScript() { // Only the first page shown runs it, not every page opened afterwards. guard !scriptedPage, let script = ProcessInfo.processInfo.environment["SNOWBOUND_SCRIPT"] else { return } @@ -35,9 +35,8 @@ extension CanvasView { var delay = 1.0 for step in script.split(separator: "|") { if step.hasPrefix("wait:") { delay += Double(step.dropFirst(5)) ?? 0 } - DispatchQueue.main.asyncAfter(deadline: .now() + delay) { [weak self] in - self?.perform(String(step)) - } + // Held, so that steps after the page goes, as Delete Page takes it, still run. + DispatchQueue.main.asyncAfter(deadline: .now() + delay) { [self] in perform(String(step)) } delay += 0.4 } } @@ -106,15 +105,45 @@ extension CanvasView { case "done": _ = resignFirstResponder() case "wait", "snap": break case "activate": - guard let found = (window?.windowScene?.windows ?? []).reversed().lazy - .compactMap({ Self.element(argument, in: $0) }).first - else { + // An alert's button, which takes no accessibility activation outside VoiceOver. + let alert = Self.windows.lazy.compactMap { window in + sequence(first: window.rootViewController, next: { $0?.presentedViewController }).compactMap { $0 }.last + }.compactMap { $0 as? UIAlertController }.first + if let alert, let action = alert.actions.first(where: { $0.title == argument }), + let handler = action.value(forKey: "handler") + { + alert.dismiss(animated: true) { + unsafeBitCast(handler as AnyObject, to: (@convention(block) (UIAlertAction) -> Void).self)(action) + } + break + } + // `Row/Button` names a button within a row. + let names = argument.split(separator: "/").map(String.init) + var found: NSObject? = Self.windows.reversed().lazy.compactMap { Self.element(names[0], in: $0) }.first + for name in names.dropFirst() { + found = found.flatMap { Self.children($0).lazy.compactMap { Self.element(name, in: $0) }.first } + } + guard let found else { NSLog("script activate: no \(argument)") break } - if let button = found as? UIButton, button.showsMenuAsPrimaryAction, let menu = button.contextMenuInteraction { + let chain = (found as? UIView).map { Array(sequence(first: $0, next: \.superview)) } ?? [] + let named = { (name: String) in chain.first { NSStringFromClass(type(of: $0)) == name } } + if let button = found as? UIControl, button.showsMenuAsPrimaryAction, let menu = button.contextMenuInteraction { let middle = CGPoint(x: button.bounds.midX, y: button.bounds.midY) menu.perform(Selector(("_presentMenuAtLocation:")), with: NSValue(cgPoint: middle)) + } else if let cell = chain.lazy.compactMap({ $0 as? UICollectionViewCell }).first, + let list = chain.lazy.compactMap({ $0 as? UICollectionView }).first, let index = list.indexPath(for: cell) + { + // A menu's item or a list's row, as a tap picks it. + if let items = named("_UIContextMenuListView"), let menu = named("_UIContextMenuView"), + let item = items.perform(Selector(("elementAtIndexPath:")), with: index)?.takeUnretainedValue() + { + menu.perform(Selector(("_handleSelectionForElement:")), with: item) + } else { + list.selectItem(at: index, animated: false, scrollPosition: []) + list.delegate?.collectionView?(list, didSelectItemAt: index) + } } else if !found.accessibilityActivate(), let control = found as? UIControl { control.sendActions(for: .primaryActionTriggered) } @@ -156,12 +185,25 @@ extension CanvasView { } extension CanvasView { - /// The first view or accessibility element under `root` VoiceOver names `label`. + /// The scene's windows, menus' and alerts' among them. + fileprivate static var windows: [UIWindow] { + UIApplication.shared.connectedScenes.compactMap { $0 as? UIWindowScene }.flatMap(\.windows) + } + + /// The frontmost view or accessibility element under `root` VoiceOver names `label`. fileprivate static func element(_ label: String, in root: NSObject) -> NSObject? { if let view = root as? UIView, view.isHidden { return nil } if root.isAccessibilityElement, root.accessibilityLabel == label { return root } - let children = ((root as? UIView)?.subviews ?? []) + (root.accessibilityElements as? [NSObject] ?? []) - return children.lazy.compactMap { element(label, in: $0) }.first + // A loop, as a lazy `first` evaluates each match twice, doubling at every level. + for child in children(root) { + if let found = element(label, in: child) { return found } + } + return nil + } + + /// `root`'s subviews, frontmost first, then its accessibility elements. + fileprivate static func children(_ root: NSObject) -> [NSObject] { + ((root as? UIView)?.subviews ?? []).reversed() + (root.accessibilityElements as? [NSObject] ?? []) } /// Logs `audit ok`, or each button of the format bar and tag grid VoiceOver can't name. -- 2.54.1