import AppKit import Foundation // MARK: - Element Model struct AXExtractedElement: Codable { let role: String let subrole: String? let title: String? let description: String? let identifier: String? let value: String? let enabled: Bool let focused: Bool let actions: [String] let position: Position? let size: Size? let children: [AXExtractedElement] let path: [Int] // Index path from app root: [0, 2, 1] = 1st child → 3rd child → 2nd child struct Position: Codable { let x: Double let y: Double } struct Size: Codable { let width: Double let height: Double } } /// Legacy output format for simulator scanning (backwards compatible) struct LegacyElement: Codable { let identifier: String let label: String let role: String let traits: [String] let frame: Frame let isEnabled: Bool let value: String? let path: [Int] let children: [LegacyElement] struct Frame: Codable { let x: Double let y: Double let width: Double let height: Double } } // MARK: - AX Attribute Helpers func getStringAttribute(_ element: AXUIElement, _ attribute: String) -> String? { var value: AnyObject? let result = AXUIElementCopyAttributeValue(element, attribute as CFString, &value) guard result == .success, let str = value as? String else { return nil } return str } func getBoolAttribute(_ element: AXUIElement, _ attribute: String) -> Bool { var value: AnyObject? let result = AXUIElementCopyAttributeValue(element, attribute as CFString, &value) guard result == .success else { return false } if let num = value as? NSNumber { return num.boolValue } return false } func getPosition(_ element: AXUIElement) -> AXExtractedElement.Position? { var posValue: AnyObject? let result = AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &posValue) guard result == .success, let val = posValue else { return nil } var position = CGPoint.zero AXValueGetValue(val as! AXValue, .cgPoint, &position) return AXExtractedElement.Position(x: Double(position.x), y: Double(position.y)) } func getSize(_ element: AXUIElement) -> AXExtractedElement.Size? { var sizeValue: AnyObject? let result = AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue) guard result == .success, let val = sizeValue else { return nil } var size = CGSize.zero AXValueGetValue(val as! AXValue, .cgSize, &size) return AXExtractedElement.Size(width: Double(size.width), height: Double(size.height)) } func getActions(_ element: AXUIElement) -> [String] { var actionsValue: CFArray? let result = AXUIElementCopyActionNames(element, &actionsValue) guard result == .success, let actions = actionsValue as? [String] else { return [] } return actions } func getChildren(_ element: AXUIElement) -> [AXUIElement] { var value: AnyObject? let result = AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as CFString, &value) guard result == .success, let children = value as? [AXUIElement] else { return [] } return children } func getFrame(_ element: AXUIElement) -> LegacyElement.Frame { let pos = getPosition(element) let sz = getSize(element) return LegacyElement.Frame( x: pos?.x ?? 0, y: pos?.y ?? 0, width: sz?.width ?? 0, height: sz?.height ?? 0 ) } // MARK: - AX Tree Walkers /// Walk element tree — new format with full AX attributes, tracking index path func walkElementFull(_ element: AXUIElement, depth: Int = 0, maxDepth: Int = 15, currentPath: [Int] = []) -> AXExtractedElement? { guard depth < maxDepth else { return nil } let role = getStringAttribute(element, kAXRoleAttribute) ?? "AXUnknown" let subrole = getStringAttribute(element, kAXSubroleAttribute) let title = getStringAttribute(element, kAXTitleAttribute) let descriptionAttr = getStringAttribute(element, kAXDescriptionAttribute) let identifier = getStringAttribute(element, "AXIdentifier") let value = getStringAttribute(element, kAXValueAttribute as String) let enabled = getBoolAttribute(element, kAXEnabledAttribute) let focused = getBoolAttribute(element, kAXFocusedAttribute) let position = getPosition(element) let size = getSize(element) let actions = getActions(element) // Skip zero-size invisible elements deep in the tree if let sz = size, sz.width <= 0 && sz.height <= 0 && depth > 2 { return nil } let axChildren = getChildren(element) var childElements: [AXExtractedElement] = [] for (index, child) in axChildren.enumerated() { let childPath = currentPath + [index] if let childEl = walkElementFull(child, depth: depth + 1, maxDepth: maxDepth, currentPath: childPath) { childElements.append(childEl) } } return AXExtractedElement( role: role, subrole: subrole, title: title, description: descriptionAttr, identifier: identifier, value: value, enabled: enabled, focused: focused, actions: actions, position: position, size: size, children: childElements, path: currentPath ) } /// Walk element tree — legacy format for simulator compatibility func walkElementLegacy(_ element: AXUIElement, depth: Int = 0, maxDepth: Int = 15, currentPath: [Int] = []) -> LegacyElement? { guard depth < maxDepth else { return nil } let role = getStringAttribute(element, kAXRoleAttribute) ?? "AXUnknown" let label = getStringAttribute(element, kAXTitleAttribute) ?? getStringAttribute(element, kAXDescriptionAttribute) ?? getStringAttribute(element, kAXValueAttribute as String) ?? "" let identifier = getStringAttribute(element, "AXIdentifier") ?? "" let isEnabled = getBoolAttribute(element, kAXEnabledAttribute) let value = getStringAttribute(element, kAXValueAttribute as String) let frame = getFrame(element) if frame.width <= 0 && frame.height <= 0 && depth > 2 { return nil } let axChildren = getChildren(element) var childElements: [LegacyElement] = [] for (index, child) in axChildren.enumerated() { let childPath = currentPath + [index] if let childEl = walkElementLegacy(child, depth: depth + 1, maxDepth: maxDepth, currentPath: childPath) { childElements.append(childEl) } } var traits: [String] = [] if role == "AXButton" { traits.append("button") } if role == "AXLink" { traits.append("link") } if role == "AXStaticText" { traits.append("staticText") } if role == "AXImage" { traits.append("image") } if !isEnabled { traits.append("disabled") } return LegacyElement( identifier: identifier, label: label, role: role, traits: traits, frame: frame, isEnabled: isEnabled, value: value, path: currentPath, children: childElements ) } // MARK: - App Finders func findSimulatorApp() -> NSRunningApplication? { let apps = NSWorkspace.shared.runningApplications return apps.first { $0.bundleIdentifier == "com.apple.iphonesimulator" } } func findSimulatorWindow(app: NSRunningApplication, deviceName: String?) -> AXUIElement? { let appElement = AXUIElementCreateApplication(app.processIdentifier) var windowsValue: AnyObject? let result = AXUIElementCopyAttributeValue(appElement, kAXWindowsAttribute as CFString, &windowsValue) guard result == .success, let windows = windowsValue as? [AXUIElement] else { return nil } if let deviceName = deviceName { for window in windows { let title = getStringAttribute(window, kAXTitleAttribute) ?? "" if title.localizedCaseInsensitiveContains(deviceName) { return window } } } return windows.first } /// R4: descend past Simulator chrome to the embedded iOS app subtree. /// /// The Simulator.app AX window's direct children include toolbar buttons /// ("Home", "Save Screen", "Rotate") AND a large container (AXGroup / /// AXScrollArea / AXLayoutArea / AXOther / AXSplitGroup) that holds the /// embedded iOS app's UI. Pre-R4, the walker started at the window and /// returned both — so `extract` would surface "Home" / "Save Screen" as /// top-level interactive elements and the iOS app's actual buttons would /// be buried (and often pruned when the walker hit its depth limit on /// chrome paths first). /// /// Strategy: among the window's children, pick the LARGEST container-role /// child whose area is ≥ 50% of the window area. That is the simulated /// device screen surface. If none qualifies (rare — e.g. window not yet /// rendered), fall back to the window itself so the caller still gets /// something. func findSimulatorAppRoot(window: AXUIElement) -> AXUIElement { let windowSize = getSize(window) let windowArea = (windowSize?.width ?? 0) * (windowSize?.height ?? 0) guard windowArea > 0 else { return window } let children = getChildren(window) let containerRoles: Set = [ "AXGroup", "AXScrollArea", "AXLayoutArea", "AXOther", "AXSplitGroup", "AXLayoutItem", ] var best: (element: AXUIElement, area: Double)? for child in children { let role = getStringAttribute(child, kAXRoleAttribute) ?? "" guard containerRoles.contains(role) else { continue } let sz = getSize(child) let area = (sz?.width ?? 0) * (sz?.height ?? 0) // Must cover at least half the window to count as the app surface guard area >= windowArea * 0.5 else { continue } if best == nil || area > best!.area { best = (child, area) } } return best?.element ?? window } /// Find a running app by name (case-insensitive substring match) func findAppByName(_ name: String) -> NSRunningApplication? { let apps = NSWorkspace.shared.runningApplications let lowered = name.lowercased() return apps.first { app in let appName = app.localizedName?.lowercased() ?? "" let bundleId = app.bundleIdentifier?.lowercased() ?? "" return appName.contains(lowered) || bundleId.contains(lowered) } } func findAppByPid(_ pid: pid_t) -> NSRunningApplication? { return NSWorkspace.shared.runningApplications.first { $0.processIdentifier == pid } } /// Find the frontmost/main window for a given app func findMainWindow(pid: pid_t) -> (window: AXUIElement, id: CGWindowID, title: String, size: CGSize)? { let appElement = AXUIElementCreateApplication(pid) var windowsValue: AnyObject? let result = AXUIElementCopyAttributeValue(appElement, kAXWindowsAttribute as CFString, &windowsValue) guard result == .success, let windows = windowsValue as? [AXUIElement], !windows.isEmpty else { return nil } // Use the frontmost/main window let window: AXUIElement var mainWindow: AnyObject? let mainResult = AXUIElementCopyAttributeValue(appElement, kAXMainWindowAttribute as CFString, &mainWindow) if mainResult == .success, let mw = mainWindow { window = (mw as! AXUIElement) } else { window = windows[0] } let title = getStringAttribute(window, kAXTitleAttribute) ?? "Untitled" let sz = getSize(window) ?? AXExtractedElement.Size(width: 800, height: 600) // Get CGWindowID via CGWindowListCopyWindowInfo var windowId: CGWindowID = 0 let windowList = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? [] for info in windowList { guard let ownerPid = info[kCGWindowOwnerPID as String] as? Int32, let wid = info[kCGWindowNumber as String] as? CGWindowID, ownerPid == pid else { continue } let wName = info[kCGWindowName as String] as? String ?? "" if wName == title || windowId == 0 { windowId = wid if wName == title { break } } } return (window: window, id: windowId, title: title, size: CGSize(width: sz.width, height: sz.height)) } // MARK: - Action Execution /// Traverse the AX tree by numeric index path from the app root func navigateToElement(root: AXUIElement, path: [Int]) -> AXUIElement? { var current = root for index in path { var childrenRef: CFTypeRef? guard AXUIElementCopyAttributeValue(current, kAXChildrenAttribute as CFString, &childrenRef) == .success, let children = childrenRef as? [AXUIElement], index < children.count else { return nil } current = children[index] } return current } /// Execute an accessibility action on the element at the given path func performAction(pid: pid_t, deviceName: String?, elementPath: [Int], action: String, value: String?) -> (Bool, String?) { let root: AXUIElement? if let app = findAppByPid(pid), app.bundleIdentifier == "com.apple.iphonesimulator", let simulatorWindow = findSimulatorWindow(app: app, deviceName: deviceName) { // R4: descend past Simulator chrome to the iOS app subtree so the // path indexes from the extractor's scan resolve to app elements // rather than toolbar buttons. root = findSimulatorAppRoot(window: simulatorWindow) } else { root = findMainWindow(pid: pid)?.window } guard let actionRoot = root else { return (false, "No action root window found for pid \(pid)") } guard let element = navigateToElement(root: actionRoot, path: elementPath) else { return (false, "Element not found at path") } switch action { case "press": let ok = AXUIElementPerformAction(element, kAXPressAction as CFString) == .success return (ok, ok ? nil : "AXPress failed") case "setValue": guard let value = value else { return (false, "setValue requires --value") } let ok = AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, value as CFTypeRef) == .success return (ok, ok ? nil : "AXSetValue failed") case "increment": let ok = AXUIElementPerformAction(element, kAXIncrementAction as CFString) == .success return (ok, ok ? nil : "AXIncrement failed") case "decrement": let ok = AXUIElementPerformAction(element, kAXDecrementAction as CFString) == .success return (ok, ok ? nil : "AXDecrement failed") case "showMenu": let ok = AXUIElementPerformAction(element, kAXShowMenuAction as CFString) == .success return (ok, ok ? nil : "AXShowMenu failed") case "confirm": let ok = AXUIElementPerformAction(element, kAXConfirmAction as CFString) == .success return (ok, ok ? nil : "AXConfirm failed") case "cancel": let ok = AXUIElementPerformAction(element, kAXCancelAction as CFString) == .success return (ok, ok ? nil : "AXCancel failed") case "focus": let ok = AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, true as CFTypeRef) == .success return (ok, ok ? nil : "AXFocus failed") case "scrollToVisible": let ok = AXUIElementPerformAction(element, "AXScrollToVisible" as CFString) == .success return (ok, ok ? nil : "AXScrollToVisible failed") default: return (false, "Unknown action: \(action)") } } // MARK: - Layout-fill / gap analysis (parity with TS analyzer) // // Optional in-Swift analysis pass over the extracted tree. Mirrors // src/native/layout-fill.ts so a caller running the binary directly (without // the IBR TS pipeline) can still get a fill / gap report. Emitted on stderr // as `LAYOUT_FINDINGS:` so it never corrupts the existing element JSON // stdout contract. Gated behind --analyze-layout; default off; fully // backward-compatible. struct LayoutFillFinding: Codable { let containerRole: String let containerLabel: String let axis: String // "horizontal" | "vertical" let emptyPx: Double let emptyPct: Double let position: String // "leading" | "between" | "trailing" let containerWidth: Double let containerHeight: Double let detail: String } func labelOfAX(_ el: AXExtractedElement) -> String { for s in [el.title, el.description, el.identifier, el.value] { if let v = s, !v.isEmpty { return v.count > 40 ? String(v.prefix(40)) + "…" : v } } return "" } func largestBandSwift(min: Double, max: Double, spans: [(Double, Double)]) -> (px: Double, position: String)? { guard !spans.isEmpty else { return nil } let sorted = spans.sorted { $0.0 < $1.0 } var merged: [(Double, Double)] = [] for s in sorted { if var last = merged.last, s.0 <= last.1 { last.1 = Swift.max(last.1, s.1) merged[merged.count - 1] = last } else { merged.append(s) } } var best: (Double, String) = (0, "leading") let leading = merged[0].0 - min if leading > best.0 { best = (leading, "leading") } for i in 1.. best.0 { best = (gap, "between") } } let trailing = max - merged[merged.count - 1].1 if trailing > best.0 { best = (trailing, "trailing") } return (best.0, best.1) } func analyzeLayoutFillSwift( _ roots: [AXExtractedElement], threshold: Double = 0.12, minContainerPx: Double = 50, maxDepth: Int = 20 ) -> [LayoutFillFinding] { var out: [LayoutFillFinding] = [] func visit(_ el: AXExtractedElement, _ depth: Int) { if depth >= maxDepth { return } if let pos = el.position, let sz = el.size, sz.width > 0, sz.height > 0 { let r = (x: pos.x, y: pos.y, w: sz.width, h: sz.height) let laid = el.children.filter { ($0.position != nil) && ($0.size != nil) && (($0.size?.width ?? 0) > 0) && (($0.size?.height ?? 0) > 0) } if !laid.isEmpty { // Horizontal if r.w >= minContainerPx { let spans: [(Double, Double)] = laid.map { let cp = $0.position!; let cs = $0.size! return (cp.x, cp.x + cs.width) } if let band = largestBandSwift(min: r.x, max: r.x + r.w, spans: spans), band.px / r.w >= threshold { let pct = band.px / r.w let lbl = labelOfAX(el) let detail = "\(el.role)\(lbl.isEmpty ? "" : " [\(lbl)]"): \(band.position) empty band " + "\(Int(band.px.rounded()))px = \(Int((pct * 100).rounded()))% of " + "container width \(Int(r.w.rounded()))px (horizontal)" out.append( LayoutFillFinding( containerRole: el.role, containerLabel: lbl, axis: "horizontal", emptyPx: band.px, emptyPct: pct, position: band.position, containerWidth: r.w, containerHeight: r.h, detail: detail )) } } // Vertical if r.h >= minContainerPx { let spans: [(Double, Double)] = laid.map { let cp = $0.position!; let cs = $0.size! return (cp.y, cp.y + cs.height) } if let band = largestBandSwift(min: r.y, max: r.y + r.h, spans: spans), band.px / r.h >= threshold { let pct = band.px / r.h let lbl = labelOfAX(el) let detail = "\(el.role)\(lbl.isEmpty ? "" : " [\(lbl)]"): \(band.position) empty band " + "\(Int(band.px.rounded()))px = \(Int((pct * 100).rounded()))% of " + "container height \(Int(r.h.rounded()))px (vertical)" out.append( LayoutFillFinding( containerRole: el.role, containerLabel: lbl, axis: "vertical", emptyPx: band.px, emptyPct: pct, position: band.position, containerWidth: r.w, containerHeight: r.h, detail: detail )) } } } } for c in el.children { visit(c, depth + 1) } } for r in roots { visit(r, 0) } return out.sorted { $0.emptyPct > $1.emptyPct } } // MARK: - Main let args = CommandLine.arguments // Parse arguments var deviceName: String? = nil var targetPid: pid_t? = nil var targetApp: String? = nil var resolveApp: String? = nil var outputMode: String = "auto" // "auto", "legacy", "full" var actionName: String? = nil var elementPathStr: String? = nil var actionValue: String? = nil var analyzeLayout: Bool = false var layoutThreshold: Double = 0.12 var i = 1 while i < args.count { switch args[i] { case "--device-name": i += 1 if i < args.count { deviceName = args[i] } case "--pid": i += 1 if i < args.count { targetPid = pid_t(args[i]) } case "--app": i += 1 if i < args.count { targetApp = args[i] } case "--resolve-app": i += 1 if i < args.count { resolveApp = args[i] } case "--format": i += 1 if i < args.count { outputMode = args[i] } case "--action": i += 1 if i < args.count { actionName = args[i] } case "--element-path": i += 1 if i < args.count { elementPathStr = args[i] } // comma-separated: "0,2,1" case "--value": i += 1 if i < args.count { actionValue = args[i] } case "--analyze-layout": analyzeLayout = true case "--layout-threshold": i += 1 if i < args.count, let t = Double(args[i]) { layoutThreshold = t } default: break } i += 1 } // --- Mode: Resolve a running macOS app without AX permission --- if let appName = resolveApp { guard let app = findAppByName(appName) else { fputs("Error: No running app found matching \"\(appName)\"\n", stderr) exit(1) } let payload: [String: Any] = [ "pid": Int(app.processIdentifier), "name": app.localizedName ?? "", "bundleIdentifier": app.bundleIdentifier ?? "" ] if let data = try? JSONSerialization.data(withJSONObject: payload), let str = String(data: data, encoding: .utf8) { print(str) exit(0) } fputs("Error: Failed to encode app resolution JSON\n", stderr) exit(1) } // Check accessibility permission let checkOpts = [kAXTrustedCheckOptionPrompt.takeUnretainedValue(): true] as CFDictionary guard AXIsProcessTrustedWithOptions(checkOpts) else { fputs("Error: Accessibility permission required. Grant access in System Settings > Privacy & Security > Accessibility\n", stderr) exit(1) } let encoder = JSONEncoder() encoder.outputFormatting = [.prettyPrinted, .sortedKeys] // --- Mode: Action execution (--action --pid --element-path) --- if let actionName = actionName, let pathStr = elementPathStr { guard let pid = targetPid else { fputs("{\"error\":\"--pid is required for action mode\"}\n", stderr) exit(1) } let path = pathStr.split(separator: ",").compactMap { Int($0) } let (success, error) = performAction(pid: pid, deviceName: deviceName, elementPath: path, action: actionName, value: actionValue) var result: [String: Any] = ["success": success, "action": actionName] if let error = error { result["error"] = error } if let data = try? JSONSerialization.data(withJSONObject: result), let str = String(data: data, encoding: .utf8) { print(str) } exit(success ? 0 : 1) } // --- Mode 1: macOS app scanning (--pid or --app) --- if targetPid != nil || targetApp != nil { let pid: pid_t if let p = targetPid { pid = p } else if let appName = targetApp { guard let app = findAppByName(appName) else { fputs("Error: No running app found matching \"\(appName)\"\n", stderr) fputs("Running apps:\n", stderr) for app in NSWorkspace.shared.runningApplications where app.activationPolicy == .regular { let name = app.localizedName ?? "Unknown" let bundle = app.bundleIdentifier ?? "unknown" fputs(" \(name) (\(bundle)) pid=\(app.processIdentifier)\n", stderr) } exit(1) } pid = app.processIdentifier } else { fputs("Error: No target specified\n", stderr) exit(1) } guard let windowInfo = findMainWindow(pid: pid) else { fputs("Error: No windows found for pid \(pid)\n", stderr) exit(1) } // Print window info header let widthInt = Int(windowInfo.size.width) let heightInt = Int(windowInfo.size.height) print("WINDOW:\(windowInfo.id):\(widthInt)x\(heightInt):\(windowInfo.title)") // Walk the accessibility tree with full format, tracking paths from the window root guard let rootElement = walkElementFull(windowInfo.window, currentPath: []) else { fputs("Error: Failed to walk accessibility tree\n", stderr) exit(1) } // Output children (each child already has its path embedded) let output = rootElement.children let data = try encoder.encode(output) if let json = String(data: data, encoding: .utf8) { print(json) } else { fputs("Error: Failed to encode JSON\n", stderr) exit(1) } // Optional layout-fill / gap analysis. Emitted on stderr as // LAYOUT_FINDINGS: so it never corrupts the stdout JSON contract. if analyzeLayout { let findings = analyzeLayoutFillSwift(output, threshold: layoutThreshold) if let fData = try? encoder.encode(findings), let fJson = String(data: fData, encoding: .utf8) { fputs("LAYOUT_FINDINGS:\(fJson)\n", stderr) } } exit(0) } // --- Mode 2: Simulator scanning (--device-name or default) --- guard let simApp = findSimulatorApp() else { fputs("Error: Simulator.app is not running\n", stderr) exit(1) } guard let window = findSimulatorWindow(app: simApp, deviceName: deviceName) else { let msg = deviceName != nil ? "Error: No simulator window found for device: \(deviceName!)" : "Error: No simulator windows found" fputs("\(msg)\n", stderr) exit(1) } // R4: descend past Simulator chrome (Home / Save Screen / Rotate buttons) // to the embedded iOS app subtree. Without this, the walker returned // chrome-tree children — the "sim_action reads simulator-chrome AX tree, // not app content" failure mode from the transcript audit. let appRoot = findSimulatorAppRoot(window: window) // Walk the accessibility tree with legacy format guard let rootElement = walkElementLegacy(appRoot, currentPath: []) else { fputs("Error: Failed to walk accessibility tree\n", stderr) exit(1) } // Output as JSON array (children of the window are the actual UI elements) let output = rootElement.children let data = try encoder.encode(output) if let json = String(data: data, encoding: .utf8) { print(json) } else { fputs("Error: Failed to encode JSON\n", stderr) exit(1) }