545 lines
21 KiB
Python
545 lines
21 KiB
Python
"""
|
||
iOS Device State Implementation
|
||
Concrete implementation of AbstractDeviceState for iOS devices.
|
||
"""
|
||
import copy
|
||
import hashlib
|
||
import os
|
||
from typing import Optional, Dict, Any, List, Set
|
||
|
||
from ...core.abstract_device_state import AbstractDeviceState
|
||
from ...core.abstract_input_event import EventType
|
||
|
||
|
||
class IOSDeviceState(AbstractDeviceState):
|
||
"""
|
||
iOS 设备状态的具体实现
|
||
|
||
通过 WDA 获取的界面信息构建设备状态。
|
||
"""
|
||
|
||
def __init__(self, device, views: List[Dict[str, Any]],
|
||
foreground_page: str = None, screenshot_path: str = None, tag: str = None):
|
||
"""
|
||
初始化 iOS 设备状态
|
||
|
||
:param device: IOSDevice 实例
|
||
:param views: 视图列表（ViewDict 格式）
|
||
:param foreground_page: 前台应用/页面标识
|
||
:param screenshot_path: 截图路径
|
||
:param tag: 状态标签
|
||
"""
|
||
super().__init__(device, tag=tag, screenshot_path=screenshot_path)
|
||
|
||
self._views = views or []
|
||
self._foreground_page = foreground_page
|
||
self._view_tree = {}
|
||
|
||
# 缓存
|
||
self._state_str_cache = None
|
||
self._structure_str_cache = None
|
||
|
||
# 需要过滤的视图 ID 集合
|
||
self._status_bar_ids: Set[int] = set()
|
||
self._off_screen_ids: Set[int] = set() # 屏幕外视图（ScrollView 中不可见部分）
|
||
|
||
# 解析视图并生成标识
|
||
self._parse_views()
|
||
|
||
# ==================== 视图信息 ====================
|
||
|
||
@property
|
||
def views(self) -> List[Dict[str, Any]]:
|
||
"""获取视图列表"""
|
||
return self._views
|
||
|
||
@property
|
||
def view_tree(self) -> Dict[str, Any]:
|
||
"""获取视图树"""
|
||
return self._view_tree
|
||
|
||
@property
|
||
def foreground_page(self) -> Optional[str]:
|
||
"""获取前台页面标识"""
|
||
return self._foreground_page
|
||
|
||
@property
|
||
def foreground_activity(self) -> Optional[str]:
|
||
"""
|
||
获取前台活动标识（兼容 Android 接口）
|
||
|
||
iOS 没有 Activity 概念，使用 foreground_page (bundle_id) 代替
|
||
"""
|
||
return self._foreground_page
|
||
|
||
@property
|
||
def search_content(self) -> str:
|
||
"""
|
||
获取用于搜索的内容（兼容 UTG 接口）
|
||
|
||
收集所有视图的文本内容用于搜索，过滤状态栏视图
|
||
"""
|
||
texts = []
|
||
for view in self._views:
|
||
# 跳过状态栏视图及其子元素、屏幕外视图
|
||
if view.get("temp_id") in self._status_bar_ids:
|
||
continue
|
||
if view.get("temp_id") in self._off_screen_ids:
|
||
continue
|
||
text = view.get("text", "")
|
||
if text:
|
||
texts.append(text)
|
||
content_desc = view.get("content_description", "")
|
||
if content_desc and content_desc != text:
|
||
texts.append(content_desc)
|
||
return " ".join(texts)
|
||
|
||
# ==================== 状态标识 ====================
|
||
|
||
@property
|
||
def state_str(self) -> str:
|
||
"""获取状态唯一标识"""
|
||
if self._state_str_cache is None:
|
||
self._state_str_cache = self._generate_state_str()
|
||
return self._state_str_cache
|
||
|
||
@property
|
||
def structure_str(self) -> str:
|
||
"""获取结构标识（忽略内容）"""
|
||
if self._structure_str_cache is None:
|
||
self._structure_str_cache = self._generate_structure_str()
|
||
return self._structure_str_cache
|
||
|
||
# ==================== 输入事件 ====================
|
||
|
||
def get_possible_input(self) -> List['AbstractInputEvent']:
|
||
"""获取可能的输入事件列表"""
|
||
# 缓存机制：如果已经计算过，直接返回缓存
|
||
# 可能导致一直不更新，先禁用
|
||
# if self._possible_events:
|
||
# return [] + self._possible_events
|
||
|
||
from .ios_input_event import (
|
||
IOSTouchEvent, IOSLongTouchEvent, IOSSwipeEvent,
|
||
IOSScrollEvent, IOSSetTextEvent, IOSKeyEvent
|
||
)
|
||
|
||
possible_events = []
|
||
enabled_view_ids = []
|
||
touch_exclude_view_ids = set()
|
||
|
||
# 预筛选：收集所有 enabled 且 visible 的视图 ID，排除屏幕外视图
|
||
for view_dict in self._views:
|
||
if view_dict.get('temp_id') in self._off_screen_ids:
|
||
continue
|
||
if (self._safe_dict_get(view_dict, 'enabled') and
|
||
self._safe_dict_get(view_dict, 'visible')):
|
||
# 检查边界是否有效
|
||
bounds = view_dict.get("bounds", [[0, 0], [0, 0]])
|
||
if bounds[0][0] < bounds[1][0] and bounds[0][1] < bounds[1][1]:
|
||
enabled_view_ids.append(view_dict['temp_id'])
|
||
|
||
# 第一轮：clickable 元素 -> 点击事件
|
||
# 并排除其所有子元素
|
||
for view_id in enabled_view_ids:
|
||
if self._safe_dict_get(self._views[view_id], 'clickable'):
|
||
view = self._views[view_id]
|
||
bounds = view.get("bounds", [[0, 0], [0, 0]])
|
||
center_x = (bounds[0][0] + bounds[1][0]) // 2
|
||
center_y = (bounds[0][1] + bounds[1][1]) // 2
|
||
possible_events.append(IOSTouchEvent(x=center_x, y=center_y, view=view))
|
||
touch_exclude_view_ids.add(view_id)
|
||
# 排除所有子元素
|
||
touch_exclude_view_ids = touch_exclude_view_ids.union(
|
||
set(self.get_all_children(self._views[view_id]))
|
||
)
|
||
|
||
# 添加通用滚动事件（类似 Android）
|
||
# 但排除系统界面，避免在系统弹窗上执行无意义的滚动
|
||
if not self._is_system_ui():
|
||
possible_events.append(IOSScrollEvent(
|
||
start_x=self.width // 2, start_y=self.height // 2 + 100,
|
||
end_x=self.width // 2, end_y=self.height // 2 - 100,
|
||
direction="up"
|
||
))
|
||
# possible_events.append(IOSScrollEvent(
|
||
# start_x=self.width // 2, start_y=self.height // 2 + 100,
|
||
# end_x=self.width // 2, end_y=self.height // 2 - 100,
|
||
# direction="up"
|
||
# ))
|
||
# possible_events.append(IOSScrollEvent(
|
||
# start_x=self.width // 2, start_y=self.height // 2 - 100,
|
||
# end_x=self.width // 2, end_y=self.height // 2 + 100,
|
||
# direction="down"
|
||
# ))
|
||
|
||
# 第二轮：iOS 特定的 scrollable 元素 -> 滚动事件
|
||
for view_id in enabled_view_ids:
|
||
if self._safe_dict_get(self._views[view_id], 'scrollable'):
|
||
view = self._views[view_id]
|
||
bounds = view.get("bounds", [[0, 0], [0, 0]])
|
||
center_x = (bounds[0][0] + bounds[1][0]) // 2
|
||
center_y = (bounds[0][1] + bounds[1][1]) // 2
|
||
|
||
possible_events.append(IOSScrollEvent(
|
||
start_x=center_x, start_y=center_y + 100,
|
||
end_x=center_x, end_y=center_y - 100,
|
||
direction="up", view=view
|
||
))
|
||
possible_events.append(IOSScrollEvent(
|
||
start_x=center_x, start_y=center_y - 100,
|
||
end_x=center_x, end_y=center_y + 100,
|
||
direction="down", view=view
|
||
))
|
||
|
||
# 第三轮：长按事件（iOS 特定类型）
|
||
for view_id in enabled_view_ids:
|
||
view = self._views[view_id]
|
||
class_name = view.get("class_name", "")
|
||
supports_long_press = class_name in ["Cell", "CollectionView", "TableView", "Link"]
|
||
|
||
if self._safe_dict_get(view, 'clickable') and supports_long_press:
|
||
bounds = view.get("bounds", [[0, 0], [0, 0]])
|
||
center_x = (bounds[0][0] + bounds[1][0]) // 2
|
||
center_y = (bounds[0][1] + bounds[1][1]) // 2
|
||
possible_events.append(IOSLongTouchEvent(x=center_x, y=center_y, view=view))
|
||
|
||
# 第四轮：editable 元素 -> 输入事件
|
||
for view_id in enabled_view_ids:
|
||
if self._safe_dict_get(self._views[view_id], 'editable'):
|
||
possible_events.append(IOSSetTextEvent(
|
||
text="cat", view=self._views[view_id]
|
||
))
|
||
touch_exclude_view_ids.add(view_id)
|
||
|
||
# 第五轮：叶子节点兜底（没有子元素且未被处理）
|
||
# 过滤明显不可交互的类型，避免生成无效点击事件
|
||
NON_INTERACTIVE_TYPES = ['StaticText', 'Image', 'Icon', 'Other', 'Indicator', 'PageIndicator']
|
||
if len(possible_events) == 1: # 仅有1个手动添加的滚动事件
|
||
for view_id in enabled_view_ids:
|
||
if view_id in touch_exclude_view_ids:
|
||
continue
|
||
children = self._safe_dict_get(self._views[view_id], 'children')
|
||
if children and len(children) > 0:
|
||
continue
|
||
|
||
view = self._views[view_id]
|
||
class_name = view.get("class_name", "")
|
||
|
||
# 跳过明显不可交互的元素类型
|
||
if any(non_interactive in class_name for non_interactive in NON_INTERACTIVE_TYPES):
|
||
continue
|
||
|
||
bounds = view.get("bounds", [[0, 0], [0, 0]])
|
||
center_x = (bounds[0][0] + bounds[1][0]) // 2
|
||
center_y = (bounds[0][1] + bounds[1][1]) // 2
|
||
possible_events.append(IOSTouchEvent(x=center_x, y=center_y, view=view))
|
||
|
||
# 缓存结果
|
||
# self._possible_events = possible_events
|
||
return [] + possible_events
|
||
|
||
@staticmethod
|
||
def _safe_dict_get(view_dict: Dict[str, Any], key: str, default=None):
|
||
"""安全获取字典值"""
|
||
value = view_dict.get(key, None)
|
||
return value if value is not None else default
|
||
|
||
# ==================== 序列化 ====================
|
||
|
||
def to_dict(self) -> Dict[str, Any]:
|
||
"""序列化为字典"""
|
||
return {
|
||
"state_str": self.state_str,
|
||
"foreground_page": self._foreground_page,
|
||
"views": self._views,
|
||
"tag": self.tag,
|
||
"screenshot_path": self.screenshot_path,
|
||
"width": self.width,
|
||
"height": self.height,
|
||
}
|
||
|
||
# ==================== 内部方法 ====================
|
||
|
||
def _parse_views(self) -> None:
|
||
"""解析视图，构建视图树"""
|
||
if not self._views:
|
||
return
|
||
|
||
# 收集需要过滤的视图 ID
|
||
self._collect_status_bar_ids()
|
||
self._collect_off_screen_ids()
|
||
|
||
# 找到根视图（parent == -1 的视图）
|
||
root_views = [v for v in self._views if v.get("parent", -1) == -1]
|
||
if root_views:
|
||
self._view_tree = self._build_tree(root_views[0])
|
||
|
||
def _collect_status_bar_ids(self) -> None:
|
||
"""收集 StatusBar 类型视图及其所有子元素的 temp_id"""
|
||
for view in self._views:
|
||
# 原始 WDA 的 type 字段在 _parse_wda_source 中被映射为 class_name
|
||
if view.get("class_name") == "StatusBar":
|
||
status_bar_id = view.get("temp_id")
|
||
if status_bar_id is not None:
|
||
self._status_bar_ids.add(status_bar_id)
|
||
# 递归收集所有子元素 ID
|
||
self._status_bar_ids.update(self.get_all_children(view))
|
||
|
||
def _collect_off_screen_ids(self) -> None:
|
||
"""
|
||
收集屏幕外视图的 temp_id
|
||
|
||
WDA source() 返回的 rect 坐标是可滚动内容的绝对位置，
|
||
超出屏幕可见范围的视图不应参与事件生成和状态标识计算。
|
||
判定标准：视图 bounds 完全位于屏幕可见区域之外。
|
||
"""
|
||
screen_w = self.width
|
||
screen_h = self.height
|
||
|
||
for view in self._views:
|
||
bounds = view.get("bounds", [[0, 0], [0, 0]])
|
||
x1, y1 = bounds[0] # 左上角
|
||
x2, y2 = bounds[1] # 右下角
|
||
|
||
# 视图完全在屏幕外（不与屏幕可见区域有任何交集）
|
||
if x2 <= 0 or x1 >= screen_w or y2 <= 0 or y1 >= screen_h:
|
||
view_id = view.get("temp_id")
|
||
if view_id is not None:
|
||
self._off_screen_ids.add(view_id)
|
||
|
||
def _build_tree(self, view: Dict) -> Dict:
|
||
"""递归构建视图树"""
|
||
tree = copy.copy(view)
|
||
children_ids = view.get("children", [])
|
||
tree["children"] = []
|
||
|
||
for child_id in children_ids:
|
||
child_view = self._get_view_by_id(child_id)
|
||
if child_view:
|
||
tree["children"].append(self._build_tree(child_view))
|
||
|
||
return tree
|
||
|
||
def _get_view_by_id(self, temp_id: int) -> Optional[Dict]:
|
||
"""根据 temp_id 获取视图"""
|
||
for view in self._views:
|
||
if view.get("temp_id") == temp_id:
|
||
return view
|
||
return None
|
||
|
||
def _generate_state_str(self) -> str:
|
||
"""生成状态唯一标识（过滤状态栏视图）"""
|
||
# 收集所有视图的签名，跳过状态栏
|
||
view_signatures = []
|
||
for view in self._views:
|
||
if view.get("temp_id") in self._status_bar_ids:
|
||
continue
|
||
if view.get("temp_id") in self._off_screen_ids:
|
||
continue
|
||
sig = self._get_view_signature(view)
|
||
view_signatures.append(sig)
|
||
|
||
# 组合前台页面和视图签名
|
||
state_content = f"{self._foreground_page or ''}_{'_'.join(sorted(view_signatures))}"
|
||
|
||
return hashlib.md5(state_content.encode()).hexdigest()
|
||
|
||
def _generate_structure_str(self) -> str:
|
||
"""生成结构标识（忽略文本内容，过滤状态栏视图）"""
|
||
view_signatures = []
|
||
for view in self._views:
|
||
if view.get("temp_id") in self._status_bar_ids:
|
||
continue
|
||
if view.get("temp_id") in self._off_screen_ids:
|
||
continue
|
||
sig = self._get_structure_signature(view)
|
||
view_signatures.append(sig)
|
||
|
||
state_content = f"{self._foreground_page or ''}_{'_'.join(sorted(view_signatures))}"
|
||
|
||
return hashlib.md5(state_content.encode()).hexdigest()
|
||
|
||
@staticmethod
|
||
def _get_view_signature(view: Dict) -> str:
|
||
"""获取视图签名（包含内容）"""
|
||
parts = [
|
||
view.get("class_name", ""),
|
||
view.get("resource_id", ""),
|
||
view.get("text", "")[:50] if view.get("text") else "",
|
||
str(view.get("bounds", [])),
|
||
]
|
||
content = "_".join(filter(None, parts))
|
||
return hashlib.md5(content.encode()).hexdigest()[:8]
|
||
|
||
@staticmethod
|
||
def _get_structure_signature(view: Dict) -> str:
|
||
"""获取结构签名（忽略文本内容）"""
|
||
parts = [
|
||
view.get("class_name", ""),
|
||
view.get("resource_id", ""),
|
||
str(view.get("bounds", [])),
|
||
]
|
||
content = "_".join(filter(None, parts))
|
||
return hashlib.md5(content.encode()).hexdigest()[:8]
|
||
|
||
# ==================== 辅助方法 ====================
|
||
|
||
def get_all_ancestors(self, view_dict: Dict[str, Any]) -> List[int]:
|
||
"""获取所有祖先节点 ID"""
|
||
ancestors = []
|
||
parent_id = view_dict.get("parent", -1)
|
||
while parent_id >= 0:
|
||
ancestors.append(parent_id)
|
||
parent_view = self._get_view_by_id(parent_id)
|
||
if parent_view:
|
||
parent_id = parent_view.get("parent", -1)
|
||
else:
|
||
break
|
||
return ancestors
|
||
|
||
def get_all_children(self, view_dict: Dict[str, Any]) -> List[int]:
|
||
"""获取所有子节点 ID（递归）"""
|
||
all_children = []
|
||
children_ids = view_dict.get("children", [])
|
||
|
||
for child_id in children_ids:
|
||
all_children.append(child_id)
|
||
child_view = self._get_view_by_id(child_id)
|
||
if child_view:
|
||
all_children.extend(self.get_all_children(child_view))
|
||
|
||
return all_children
|
||
|
||
def _is_system_ui(self) -> bool:
|
||
"""
|
||
检查当前是否是系统界面
|
||
|
||
Returns:
|
||
bool: True 表示系统界面（如 springboard），False 表示应用界面
|
||
"""
|
||
return self._foreground_page == "com.apple.springboard"
|
||
|
||
def get_app_page_depth(self) -> int:
|
||
"""获取应用页面深度（iOS 不支持，返回 -1）"""
|
||
# iOS 没有 Android 那样的 Activity 栈概念
|
||
return -1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
"""
|
||
测试屏幕外视图过滤效果
|
||
用法: cd autool && python -m DroidBot.platforms.ios.ios_device_state
|
||
"""
|
||
import json
|
||
import sys
|
||
import os
|
||
|
||
# 测试用的两个状态文件
|
||
TEST_FILES = [
|
||
("State1（欢迎页）", "output/ios_test_20260316_103839/co_vulcanlabs_moodtracker_iOS_20260316_103846/droidbot/states/state_2026-03-16_103924.json"),
|
||
("State2（评论列表页）", "output/ios_test_20260316_103839/co_vulcanlabs_moodtracker_iOS_20260316_103846/droidbot/states/state_2026-03-16_104804.json"),
|
||
]
|
||
|
||
# Mock device 对象，提供 width/height
|
||
class MockDevice:
|
||
def __init__(self, width, height):
|
||
self._width = width
|
||
self._height = height
|
||
self.output_dir = None
|
||
def get_width(self):
|
||
return self._width
|
||
def get_height(self):
|
||
return self._height
|
||
def get_display_info(self):
|
||
return {"width": self._width, "height": self._height}
|
||
|
||
# 项目根目录
|
||
project_root = os.path.dirname(os.path.abspath(__file__))
|
||
while not os.path.exists(os.path.join(project_root, "output")) and project_root != "/":
|
||
project_root = os.path.dirname(project_root)
|
||
|
||
for label, rel_path in TEST_FILES:
|
||
json_path = os.path.join(project_root, rel_path)
|
||
if not os.path.exists(json_path):
|
||
print(f"⚠️ {label}: 文件不存在 {json_path}")
|
||
continue
|
||
|
||
data = json.load(open(json_path))
|
||
views = data["views"]
|
||
width = data.get("width", 390)
|
||
height = data.get("height", 844)
|
||
foreground_page = data.get("foreground_page", "")
|
||
|
||
# 创建 Mock device 并构建 IOSDeviceState
|
||
device = MockDevice(width, height)
|
||
state = IOSDeviceState(
|
||
device=device,
|
||
views=views,
|
||
foreground_page=foreground_page,
|
||
)
|
||
|
||
# 统计信息
|
||
total_views = len(views)
|
||
off_screen_count = len(state._off_screen_ids)
|
||
status_bar_count = len(state._status_bar_ids)
|
||
on_screen_count = total_views - off_screen_count - status_bar_count
|
||
|
||
print(f"\n{'='*60}")
|
||
print(f"📱 {label}")
|
||
print(f"{'='*60}")
|
||
print(f"屏幕尺寸: {width}x{height}")
|
||
print(f"前台页面: {foreground_page}")
|
||
print(f"总视图数: {total_views}")
|
||
print(f" 状态栏视图: {status_bar_count}")
|
||
print(f" 屏幕外视图: {off_screen_count}")
|
||
print(f" 有效视图: {on_screen_count}")
|
||
|
||
# 获取 possible input 事件
|
||
events = state.get_possible_input()
|
||
touch_events = [e for e in events if e.__class__.__name__ == "IOSTouchEvent"]
|
||
long_touch_events = [e for e in events if e.__class__.__name__ == "IOSLongTouchEvent"]
|
||
scroll_events = [e for e in events if e.__class__.__name__ == "IOSScrollEvent"]
|
||
set_text_events = [e for e in events if e.__class__.__name__ == "IOSSetTextEvent"]
|
||
|
||
print(f"\n📋 可执行事件 (共 {len(events)} 个):")
|
||
print(f" Touch: {len(touch_events)}")
|
||
print(f" LongTouch: {len(long_touch_events)}")
|
||
print(f" Scroll: {len(scroll_events)}")
|
||
print(f" SetText: {len(set_text_events)}")
|
||
|
||
# 验证所有 touch 事件坐标在屏幕范围内
|
||
out_of_bounds = []
|
||
for e in touch_events + long_touch_events:
|
||
if e.x < 0 or e.x > width or e.y < 0 or e.y > height:
|
||
out_of_bounds.append(e)
|
||
|
||
if out_of_bounds:
|
||
print(f"\n❌ 发现 {len(out_of_bounds)} 个坐标越界事件:")
|
||
for e in out_of_bounds[:5]:
|
||
print(f" {e.get_event_str()}")
|
||
else:
|
||
print(f"\n✅ 所有触摸/长按事件坐标均在屏幕范围内")
|
||
|
||
# 打印 touch 事件详情
|
||
print(f"\n📍 Touch 事件列表:")
|
||
for e in touch_events:
|
||
view = e.view
|
||
class_name = view.get("class_name", "?") if view else "?"
|
||
text = (view.get("text", "") or "")[:30] if view else ""
|
||
text_str = f' text="{text}"' if text else ""
|
||
print(f" ({e.x:4d}, {e.y:4d}) [{class_name}]{text_str}")
|
||
|
||
# 打印 long touch 事件详情
|
||
if long_touch_events:
|
||
print(f"\n📍 LongTouch 事件列表:")
|
||
for e in long_touch_events:
|
||
view = e.view
|
||
class_name = view.get("class_name", "?") if view else "?"
|
||
text = (view.get("text", "") or "")[:30] if view else ""
|
||
text_str = f' text="{text}"' if text else ""
|
||
print(f" ({e.x:4d}, {e.y:4d}) [{class_name}]{text_str}")
|
||
|
||
print(f"\n🔑 state_str: {state.state_str}")
|
||
print(f"🔑 structure_str: {state.structure_str}")
|