208 lines
7.3 KiB
Python
208 lines
7.3 KiB
Python
"""
|
||
Windows Device State Implementation
|
||
Concrete implementation of AbstractDeviceState for Windows desktop applications.
|
||
Uses CV mode for UI element detection.
|
||
"""
|
||
import hashlib
|
||
from typing import Optional, Dict, Any, List
|
||
|
||
from ...core.abstract_device_state import AbstractDeviceState
|
||
from ...core.abstract_input_event import EventType
|
||
|
||
|
||
class WindowsDeviceState(AbstractDeviceState):
|
||
"""
|
||
Windows 设备状态的具体实现（CV 模式）
|
||
|
||
使用 OmniParser 进行 UI 元素检测，无需 UIAutomation。
|
||
"""
|
||
|
||
def __init__(self, device, cv_views: List[Dict[str, Any]],
|
||
window_title: str = None, tag: str = None,
|
||
screenshot_path: str = None):
|
||
"""
|
||
初始化 Windows 设备状态
|
||
|
||
:param device: WindowsDevice 实例
|
||
:param cv_views: CV 检测得到的 ViewDict 列表
|
||
:param window_title: 当前窗口标题
|
||
:param tag: 状态标签
|
||
:param screenshot_path: 截图路径
|
||
"""
|
||
super().__init__(device, tag, screenshot_path)
|
||
|
||
self._cv_views = cv_views or []
|
||
self._window_title = window_title
|
||
|
||
# 生成视图字符串标识
|
||
self._generate_view_strs()
|
||
|
||
# 缓存
|
||
self._state_str = None
|
||
self._structure_str = None
|
||
|
||
# ==================== 视图信息 ====================
|
||
|
||
@property
|
||
def views(self) -> List[Dict[str, Any]]:
|
||
"""获取 CV 检测的视图列表"""
|
||
return self._cv_views
|
||
|
||
@property
|
||
def cv_views(self) -> List[Dict[str, Any]]:
|
||
"""获取 CV 视图列表（与 views 相同）"""
|
||
return self._cv_views
|
||
|
||
# ==================== 状态标识 ====================
|
||
|
||
@property
|
||
def state_str(self) -> str:
|
||
"""获取状态的唯一标识字符串"""
|
||
if self._state_str is None:
|
||
self._state_str = self._get_state_str()
|
||
return self._state_str
|
||
|
||
@property
|
||
def structure_str(self) -> str:
|
||
"""获取状态的结构标识（忽略内容）"""
|
||
if self._structure_str is None:
|
||
self._structure_str = self._get_content_free_state_str()
|
||
return self._structure_str
|
||
|
||
@property
|
||
def foreground_page(self) -> Optional[str]:
|
||
"""返回当前窗口标题作为页面标识"""
|
||
return self._window_title
|
||
|
||
@property
|
||
def search_content(self) -> str:
|
||
"""获取用于搜索的文本内容"""
|
||
texts = []
|
||
for view in self._cv_views:
|
||
text = view.get('text', '')
|
||
if text:
|
||
texts.append(text)
|
||
return ' '.join(texts)
|
||
|
||
# ==================== 私有方法 ====================
|
||
|
||
def _generate_view_strs(self):
|
||
"""为每个视图生成唯一标识符"""
|
||
for idx, view in enumerate(self._cv_views):
|
||
if 'view_str' not in view or not view['view_str']:
|
||
bounds = view.get('bounds', [[0, 0], [0, 0]])
|
||
x1, y1 = bounds[0]
|
||
x2, y2 = bounds[1]
|
||
text = view.get('text', '')[:20] # 截取前20个字符
|
||
view['view_str'] = f"win_cv_{idx}_{x1}_{y1}_{x2}_{y2}_{text}"
|
||
|
||
def _get_state_str(self) -> str:
|
||
"""生成状态唯一标识"""
|
||
state_raw = self._get_state_str_raw()
|
||
return hashlib.md5(state_raw.encode('utf-8')).hexdigest()
|
||
|
||
def _get_state_str_raw(self) -> str:
|
||
"""获取原始状态字符串"""
|
||
view_signatures = []
|
||
for view in self._cv_views:
|
||
sig = self._get_view_signature(view)
|
||
view_signatures.append(sig)
|
||
|
||
view_signatures.sort()
|
||
state_str = f"window={self._window_title}&views=" + ','.join(view_signatures)
|
||
return state_str
|
||
|
||
def _get_content_free_state_str(self) -> str:
|
||
"""获取内容无关的状态标识"""
|
||
view_signatures = []
|
||
for view in self._cv_views:
|
||
sig = self._get_content_free_view_signature(view)
|
||
view_signatures.append(sig)
|
||
|
||
view_signatures.sort()
|
||
state_str = f"window={self._window_title}&structure=" + ','.join(view_signatures)
|
||
return hashlib.md5(state_str.encode('utf-8')).hexdigest()
|
||
|
||
@staticmethod
|
||
def _get_view_signature(view_dict: Dict[str, Any]) -> str:
|
||
"""获取视图签名（包含内容）"""
|
||
# 如果已有签名则直接返回（缓存）
|
||
if 'signature' in view_dict:
|
||
return view_dict['signature']
|
||
|
||
bounds = view_dict.get('bounds', [[0, 0], [0, 0]])
|
||
text = view_dict.get('text', '')
|
||
class_name = view_dict.get('class_name', 'unknown')
|
||
clickable = view_dict.get('clickable', False)
|
||
|
||
signature = f"{class_name}:{bounds}:{text}:{clickable}"
|
||
view_dict['signature'] = signature # 存回字典供其他地方使用
|
||
return signature
|
||
|
||
@staticmethod
|
||
def _get_content_free_view_signature(view_dict: Dict[str, Any]) -> str:
|
||
"""获取内容无关的视图签名"""
|
||
bounds = view_dict.get('bounds', [[0, 0], [0, 0]])
|
||
class_name = view_dict.get('class_name', 'unknown')
|
||
clickable = view_dict.get('clickable', False)
|
||
|
||
# 只使用位置和类型，忽略文本内容
|
||
return f"{class_name}:{bounds}:{clickable}"
|
||
|
||
# ==================== 输入事件 ====================
|
||
|
||
def get_possible_input(self) -> List:
|
||
"""获取当前状态可能的输入事件列表"""
|
||
from .windows_input_event import (
|
||
WindowsTouchEvent, WindowsScrollEvent, WindowsKeyEvent
|
||
)
|
||
|
||
possible_events = []
|
||
|
||
# 为每个可点击的视图生成点击事件
|
||
for view in self._cv_views:
|
||
if view.get('clickable', False) and view.get('visible', True):
|
||
touch_event = WindowsTouchEvent(view=view)
|
||
possible_events.append(touch_event)
|
||
|
||
# 添加滚动事件
|
||
possible_events.append(WindowsScrollEvent(direction='up'))
|
||
possible_events.append(WindowsScrollEvent(direction='down'))
|
||
|
||
# 添加常用按键事件
|
||
possible_events.append(WindowsKeyEvent('ESCAPE'))
|
||
possible_events.append(WindowsKeyEvent('ENTER'))
|
||
|
||
return possible_events
|
||
|
||
# ==================== 序列化 ====================
|
||
|
||
def to_dict(self) -> Dict[str, Any]:
|
||
"""序列化为字典"""
|
||
return {
|
||
'tag': self.tag,
|
||
'window_title': self._window_title,
|
||
'state_str': self.state_str,
|
||
'structure_str': self.structure_str,
|
||
'foreground_page': self.foreground_page,
|
||
'views': self._cv_views,
|
||
'screenshot_path': self.screenshot_path,
|
||
'width': self.width,
|
||
'height': self.height,
|
||
}
|
||
|
||
# ==================== 应用信息 ====================
|
||
|
||
def get_app_page_depth(self) -> int:
|
||
"""
|
||
获取应用页面深度
|
||
|
||
Windows 应用通常没有明确的页面栈概念。
|
||
返回 0 表示应用处于正常状态（在应用内），
|
||
这样 MemoryGuidedPolicy 的记忆学习和卡住检测才能正常工作。
|
||
|
||
注意：返回 -1 会导致 Memory._memorize_state() 跳过学习，
|
||
且 input_policy.py 中的 stuck 检测也会被跳过。
|
||
"""
|
||
return 0
|