""" iOS Device State Implementation Concrete implementation of AbstractDeviceState for iOS devices. """ import copy import hashlib import os from typing import Optional, Dict, Any, List, Set from ...core.abstract_device_state import AbstractDeviceState from ...core.abstract_input_event import EventType class IOSDeviceState(AbstractDeviceState): """ iOS 设备状态的具体实现 通过 WDA 获取的界面信息构建设备状态。 """ def __init__(self, device, views: List[Dict[str, Any]], foreground_page: str = None, screenshot_path: str = None, tag: str = None): """ 初始化 iOS 设备状态 :param device: IOSDevice 实例 :param views: 视图列表（ViewDict 格式） :param foreground_page: 前台应用/页面标识 :param screenshot_path: 截图路径 :param tag: 状态标签 """ super().__init__(device, tag=tag, screenshot_path=screenshot_path) self._views = views or [] self._foreground_page = foreground_page self._view_tree = {} # 缓存 self._state_str_cache = None self._structure_str_cache = None # 需要过滤的视图 ID 集合 self._status_bar_ids: Set[int] = set() self._off_screen_ids: Set[int] = set() # 屏幕外视图（ScrollView 中不可见部分） # 解析视图并生成标识 self._parse_views() # ==================== 视图信息 ==================== @property def views(self) -> List[Dict[str, Any]]: """获取视图列表""" return self._views @property def view_tree(self) -> Dict[str, Any]: """获取视图树""" return self._view_tree @property def foreground_page(self) -> Optional[str]: """获取前台页面标识""" return self._foreground_page @property def foreground_activity(self) -> Optional[str]: """ 获取前台活动标识（兼容 Android 接口） iOS 没有 Activity 概念，使用 foreground_page (bundle_id) 代替 """ return self._foreground_page @property def search_content(self) -> str: """ 获取用于搜索的内容（兼容 UTG 接口） 收集所有视图的文本内容用于搜索，过滤状态栏视图 """ texts = [] for view in self._views: # 跳过状态栏视图及其子元素、屏幕外视图 if view.get("temp_id") in self._status_bar_ids: continue if view.get("temp_id") in self._off_screen_ids: continue text = view.get("text", "") if text: texts.append(text) content_desc = view.get("content_description", "") if content_desc and content_desc != text: texts.append(content_desc) return " ".join(texts) # ==================== 状态标识 ==================== @property def state_str(self) -> str: """获取状态唯一标识""" if self._state_str_cache is None: self._state_str_cache = self._generate_state_str() return self._state_str_cache @property def structure_str(self) -> str: """获取结构标识（忽略内容）""" if self._structure_str_cache is None: self._structure_str_cache = self._generate_structure_str() return self._structure_str_cache # ==================== 输入事件 ==================== def get_possible_input(self) -> List['AbstractInputEvent']: """获取可能的输入事件列表""" # 缓存机制：如果已经计算过，直接返回缓存 # 可能导致一直不更新，先禁用 # if self._possible_events: # return [] + self._possible_events from .ios_input_event import ( IOSTouchEvent, IOSLongTouchEvent, IOSSwipeEvent, IOSScrollEvent, IOSSetTextEvent, IOSKeyEvent ) possible_events = [] enabled_view_ids = [] touch_exclude_view_ids = set() # 预筛选：收集所有 enabled 且 visible 的视图 ID，排除屏幕外视图 for view_dict in self._views: if view_dict.get('temp_id') in self._off_screen_ids: continue if (self._safe_dict_get(view_dict, 'enabled') and self._safe_dict_get(view_dict, 'visible')): # 检查边界是否有效 bounds = view_dict.get("bounds", [[0, 0], [0, 0]]) if bounds[0][0] < bounds[1][0] and bounds[0][1] < bounds[1][1]: enabled_view_ids.append(view_dict['temp_id']) # 第一轮：clickable 元素 -> 点击事件 # 并排除其所有子元素 for view_id in enabled_view_ids: if self._safe_dict_get(self._views[view_id], 'clickable'): view = self._views[view_id] bounds = view.get("bounds", [[0, 0], [0, 0]]) center_x = (bounds[0][0] + bounds[1][0]) // 2 center_y = (bounds[0][1] + bounds[1][1]) // 2 possible_events.append(IOSTouchEvent(x=center_x, y=center_y, view=view)) touch_exclude_view_ids.add(view_id) # 排除所有子元素 touch_exclude_view_ids = touch_exclude_view_ids.union( set(self.get_all_children(self._views[view_id])) ) # 添加通用滚动事件（类似 Android） # 但排除系统界面，避免在系统弹窗上执行无意义的滚动 if not self._is_system_ui(): possible_events.append(IOSScrollEvent( start_x=self.width // 2, start_y=self.height // 2 + 100, end_x=self.width // 2, end_y=self.height // 2 - 100, direction="up" )) # possible_events.append(IOSScrollEvent( # start_x=self.width // 2, start_y=self.height // 2 + 100, # end_x=self.width // 2, end_y=self.height // 2 - 100, # direction="up" # )) # possible_events.append(IOSScrollEvent( # start_x=self.width // 2, start_y=self.height // 2 - 100, # end_x=self.width // 2, end_y=self.height // 2 + 100, # direction="down" # )) # 第二轮：iOS 特定的 scrollable 元素 -> 滚动事件 for view_id in enabled_view_ids: if self._safe_dict_get(self._views[view_id], 'scrollable'): view = self._views[view_id] bounds = view.get("bounds", [[0, 0], [0, 0]]) center_x = (bounds[0][0] + bounds[1][0]) // 2 center_y = (bounds[0][1] + bounds[1][1]) // 2 possible_events.append(IOSScrollEvent( start_x=center_x, start_y=center_y + 100, end_x=center_x, end_y=center_y - 100, direction="up", view=view )) possible_events.append(IOSScrollEvent( start_x=center_x, start_y=center_y - 100, end_x=center_x, end_y=center_y + 100, direction="down", view=view )) # 第三轮：长按事件（iOS 特定类型） for view_id in enabled_view_ids: view = self._views[view_id] class_name = view.get("class_name", "") supports_long_press = class_name in ["Cell", "CollectionView", "TableView", "Link"] if self._safe_dict_get(view, 'clickable') and supports_long_press: bounds = view.get("bounds", [[0, 0], [0, 0]]) center_x = (bounds[0][0] + bounds[1][0]) // 2 center_y = (bounds[0][1] + bounds[1][1]) // 2 possible_events.append(IOSLongTouchEvent(x=center_x, y=center_y, view=view)) # 第四轮：editable 元素 -> 输入事件 for view_id in enabled_view_ids: if self._safe_dict_get(self._views[view_id], 'editable'): possible_events.append(IOSSetTextEvent( text="cat", view=self._views[view_id] )) touch_exclude_view_ids.add(view_id) # 第五轮：叶子节点兜底（没有子元素且未被处理） # 过滤明显不可交互的类型，避免生成无效点击事件 NON_INTERACTIVE_TYPES = ['StaticText', 'Image', 'Icon', 'Other', 'Indicator', 'PageIndicator'] if len(possible_events) == 1: # 仅有1个手动添加的滚动事件 for view_id in enabled_view_ids: if view_id in touch_exclude_view_ids: continue children = self._safe_dict_get(self._views[view_id], 'children') if children and len(children) > 0: continue view = self._views[view_id] class_name = view.get("class_name", "") # 跳过明显不可交互的元素类型 if any(non_interactive in class_name for non_interactive in NON_INTERACTIVE_TYPES): continue bounds = view.get("bounds", [[0, 0], [0, 0]]) center_x = (bounds[0][0] + bounds[1][0]) // 2 center_y = (bounds[0][1] + bounds[1][1]) // 2 possible_events.append(IOSTouchEvent(x=center_x, y=center_y, view=view)) # 缓存结果 # self._possible_events = possible_events return [] + possible_events @staticmethod def _safe_dict_get(view_dict: Dict[str, Any], key: str, default=None): """安全获取字典值""" value = view_dict.get(key, None) return value if value is not None else default # ==================== 序列化 ==================== def to_dict(self) -> Dict[str, Any]: """序列化为字典""" return { "state_str": self.state_str, "foreground_page": self._foreground_page, "views": self._views, "tag": self.tag, "screenshot_path": self.screenshot_path, "width": self.width, "height": self.height, } # ==================== 内部方法 ==================== def _parse_views(self) -> None: """解析视图，构建视图树""" if not self._views: return # 收集需要过滤的视图 ID self._collect_status_bar_ids() self._collect_off_screen_ids() # 找到根视图（parent == -1 的视图） root_views = [v for v in self._views if v.get("parent", -1) == -1] if root_views: self._view_tree = self._build_tree(root_views[0]) def _collect_status_bar_ids(self) -> None: """收集 StatusBar 类型视图及其所有子元素的 temp_id""" for view in self._views: # 原始 WDA 的 type 字段在 _parse_wda_source 中被映射为 class_name if view.get("class_name") == "StatusBar": status_bar_id = view.get("temp_id") if status_bar_id is not None: self._status_bar_ids.add(status_bar_id) # 递归收集所有子元素 ID self._status_bar_ids.update(self.get_all_children(view)) def _collect_off_screen_ids(self) -> None: """ 收集屏幕外视图的 temp_id WDA source() 返回的 rect 坐标是可滚动内容的绝对位置， 超出屏幕可见范围的视图不应参与事件生成和状态标识计算。 判定标准：视图 bounds 完全位于屏幕可见区域之外。 """ screen_w = self.width screen_h = self.height for view in self._views: bounds = view.get("bounds", [[0, 0], [0, 0]]) x1, y1 = bounds[0] # 左上角 x2, y2 = bounds[1] # 右下角 # 视图完全在屏幕外（不与屏幕可见区域有任何交集） if x2 <= 0 or x1 >= screen_w or y2 <= 0 or y1 >= screen_h: view_id = view.get("temp_id") if view_id is not None: self._off_screen_ids.add(view_id) def _build_tree(self, view: Dict) -> Dict: """递归构建视图树""" tree = copy.copy(view) children_ids = view.get("children", []) tree["children"] = [] for child_id in children_ids: child_view = self._get_view_by_id(child_id) if child_view: tree["children"].append(self._build_tree(child_view)) return tree def _get_view_by_id(self, temp_id: int) -> Optional[Dict]: """根据 temp_id 获取视图""" for view in self._views: if view.get("temp_id") == temp_id: return view return None def _generate_state_str(self) -> str: """生成状态唯一标识（过滤状态栏视图）""" # 收集所有视图的签名，跳过状态栏 view_signatures = [] for view in self._views: if view.get("temp_id") in self._status_bar_ids: continue if view.get("temp_id") in self._off_screen_ids: continue sig = self._get_view_signature(view) view_signatures.append(sig) # 组合前台页面和视图签名 state_content = f"{self._foreground_page or ''}_{'_'.join(sorted(view_signatures))}" return hashlib.md5(state_content.encode()).hexdigest() def _generate_structure_str(self) -> str: """生成结构标识（忽略文本内容，过滤状态栏视图）""" view_signatures = [] for view in self._views: if view.get("temp_id") in self._status_bar_ids: continue if view.get("temp_id") in self._off_screen_ids: continue sig = self._get_structure_signature(view) view_signatures.append(sig) state_content = f"{self._foreground_page or ''}_{'_'.join(sorted(view_signatures))}" return hashlib.md5(state_content.encode()).hexdigest() @staticmethod def _get_view_signature(view: Dict) -> str: """获取视图签名（包含内容）""" parts = [ view.get("class_name", ""), view.get("resource_id", ""), view.get("text", "")[:50] if view.get("text") else "", str(view.get("bounds", [])), ] content = "_".join(filter(None, parts)) return hashlib.md5(content.encode()).hexdigest()[:8] @staticmethod def _get_structure_signature(view: Dict) -> str: """获取结构签名（忽略文本内容）""" parts = [ view.get("class_name", ""), view.get("resource_id", ""), str(view.get("bounds", [])), ] content = "_".join(filter(None, parts)) return hashlib.md5(content.encode()).hexdigest()[:8] # ==================== 辅助方法 ==================== def get_all_ancestors(self, view_dict: Dict[str, Any]) -> List[int]: """获取所有祖先节点 ID""" ancestors = [] parent_id = view_dict.get("parent", -1) while parent_id >= 0: ancestors.append(parent_id) parent_view = self._get_view_by_id(parent_id) if parent_view: parent_id = parent_view.get("parent", -1) else: break return ancestors def get_all_children(self, view_dict: Dict[str, Any]) -> List[int]: """获取所有子节点 ID（递归）""" all_children = [] children_ids = view_dict.get("children", []) for child_id in children_ids: all_children.append(child_id) child_view = self._get_view_by_id(child_id) if child_view: all_children.extend(self.get_all_children(child_view)) return all_children def _is_system_ui(self) -> bool: """ 检查当前是否是系统界面 Returns: bool: True 表示系统界面（如 springboard），False 表示应用界面 """ return self._foreground_page == "com.apple.springboard" def get_app_page_depth(self) -> int: """获取应用页面深度（iOS 不支持，返回 -1）""" # iOS 没有 Android 那样的 Activity 栈概念 return -1 if __name__ == "__main__": """ 测试屏幕外视图过滤效果 用法: cd autool && python -m DroidBot.platforms.ios.ios_device_state """ import json import sys import os # 测试用的两个状态文件 TEST_FILES = [ ("State1（欢迎页）", "output/ios_test_20260316_103839/co_vulcanlabs_moodtracker_iOS_20260316_103846/droidbot/states/state_2026-03-16_103924.json"), ("State2（评论列表页）", "output/ios_test_20260316_103839/co_vulcanlabs_moodtracker_iOS_20260316_103846/droidbot/states/state_2026-03-16_104804.json"), ] # Mock device 对象，提供 width/height class MockDevice: def __init__(self, width, height): self._width = width self._height = height self.output_dir = None def get_width(self): return self._width def get_height(self): return self._height def get_display_info(self): return {"width": self._width, "height": self._height} # 项目根目录 project_root = os.path.dirname(os.path.abspath(__file__)) while not os.path.exists(os.path.join(project_root, "output")) and project_root != "/": project_root = os.path.dirname(project_root) for label, rel_path in TEST_FILES: json_path = os.path.join(project_root, rel_path) if not os.path.exists(json_path): print(f"⚠️ {label}: 文件不存在 {json_path}") continue data = json.load(open(json_path)) views = data["views"] width = data.get("width", 390) height = data.get("height", 844) foreground_page = data.get("foreground_page", "") # 创建 Mock device 并构建 IOSDeviceState device = MockDevice(width, height) state = IOSDeviceState( device=device, views=views, foreground_page=foreground_page, ) # 统计信息 total_views = len(views) off_screen_count = len(state._off_screen_ids) status_bar_count = len(state._status_bar_ids) on_screen_count = total_views - off_screen_count - status_bar_count print(f"\n{'='*60}") print(f"📱 {label}") print(f"{'='*60}") print(f"屏幕尺寸: {width}x{height}") print(f"前台页面: {foreground_page}") print(f"总视图数: {total_views}") print(f" 状态栏视图: {status_bar_count}") print(f" 屏幕外视图: {off_screen_count}") print(f" 有效视图: {on_screen_count}") # 获取 possible input 事件 events = state.get_possible_input() touch_events = [e for e in events if e.__class__.__name__ == "IOSTouchEvent"] long_touch_events = [e for e in events if e.__class__.__name__ == "IOSLongTouchEvent"] scroll_events = [e for e in events if e.__class__.__name__ == "IOSScrollEvent"] set_text_events = [e for e in events if e.__class__.__name__ == "IOSSetTextEvent"] print(f"\n📋 可执行事件 (共 {len(events)} 个):") print(f" Touch: {len(touch_events)}") print(f" LongTouch: {len(long_touch_events)}") print(f" Scroll: {len(scroll_events)}") print(f" SetText: {len(set_text_events)}") # 验证所有 touch 事件坐标在屏幕范围内 out_of_bounds = [] for e in touch_events + long_touch_events: if e.x < 0 or e.x > width or e.y < 0 or e.y > height: out_of_bounds.append(e) if out_of_bounds: print(f"\n❌ 发现 {len(out_of_bounds)} 个坐标越界事件:") for e in out_of_bounds[:5]: print(f" {e.get_event_str()}") else: print(f"\n✅ 所有触摸/长按事件坐标均在屏幕范围内") # 打印 touch 事件详情 print(f"\n📍 Touch 事件列表:") for e in touch_events: view = e.view class_name = view.get("class_name", "?") if view else "?" text = (view.get("text", "") or "")[:30] if view else "" text_str = f' text="{text}"' if text else "" print(f" ({e.x:4d}, {e.y:4d}) [{class_name}]{text_str}") # 打印 long touch 事件详情 if long_touch_events: print(f"\n📍 LongTouch 事件列表:") for e in long_touch_events: view = e.view class_name = view.get("class_name", "?") if view else "?" text = (view.get("text", "") or "")[:30] if view else "" text_str = f' text="{text}"' if text else "" print(f" ({e.x:4d}, {e.y:4d}) [{class_name}]{text_str}") print(f"\n🔑 state_str: {state.state_str}") print(f"🔑 structure_str: {state.structure_str}")