767 lines
30 KiB
Python
767 lines
30 KiB
Python
"""
|
||
Decision Maker - Core GuiAgent decision engine without device coupling
|
||
|
||
Extracted from GuiAgent/core/agent.py, this module provides stateless LLM-based
|
||
decision making for GUI automation without managing device connections.
|
||
"""
|
||
import base64
|
||
import logging
|
||
import json
|
||
import time
|
||
import os
|
||
from pathlib import Path
|
||
from typing import Dict, Any, Tuple, Optional
|
||
from PIL import Image
|
||
import sys
|
||
|
||
# 智能导入:先尝试相对导入,失败则使用绝对导入
|
||
# 支持:模块导入、直接运行、调试器运行等多种场景
|
||
try:
|
||
from .context_manager import ContextManager
|
||
from .llm_client import LLMClient
|
||
from .prompt_builder import get_ios_prompt, get_android_prompt, get_desktop_prompt, get_verification_prompt
|
||
from .utils import parse_uitars_action, convert_to_executor_action, strip_base64_prefix, draw_grid_on_image, draw_last_action_marker, image_to_base64
|
||
from .constants import is_absolute_coord_model, CURRENT_MODEL_NAME
|
||
except ImportError:
|
||
# 相对导入失败,添加项目根目录到 sys.path 并使用绝对导入
|
||
current_file = Path(__file__).resolve()
|
||
project_root = current_file.parent.parent.parent
|
||
if str(project_root) not in sys.path:
|
||
sys.path.insert(0, str(project_root))
|
||
from DroidBot.guiagent_core.context_manager import ContextManager
|
||
from DroidBot.guiagent_core.llm_client import LLMClient
|
||
from DroidBot.guiagent_core.prompt_builder import get_ios_prompt, get_android_prompt, get_desktop_prompt, get_verification_prompt
|
||
from DroidBot.guiagent_core.utils import parse_uitars_action, convert_to_executor_action, strip_base64_prefix, draw_grid_on_image, draw_last_action_marker, image_to_base64
|
||
from DroidBot.guiagent_core.constants import is_absolute_coord_model, CURRENT_MODEL_NAME
|
||
|
||
import re
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class GuiAgentDecisionMaker:
|
||
"""
|
||
GuiAgent 核心决策引擎(无状态,不管理设备)
|
||
|
||
提供基于LLM的GUI自动化决策能力,支持多平台(iOS, Android, Desktop)。
|
||
专为DroidBot等自动化框架设计,实现决策与执行的解耦。
|
||
"""
|
||
|
||
# 全局步数计数器,用于记录本轮采集时的GUI Agent步数总和
|
||
total_steps = 0
|
||
|
||
def __init__(
|
||
self,
|
||
platform: str,
|
||
resolution: Tuple[int, int] = None,
|
||
scale: float = 1.0,
|
||
absolute_mode: bool = None
|
||
):
|
||
"""
|
||
初始化决策引擎
|
||
|
||
Args:
|
||
platform: 平台类型 ("ios", "android", "desktop")
|
||
resolution: 屏幕分辨率 (width, height)
|
||
对于iOS: 这是截图的物理像素尺寸 (e.g., 1125x2436 for iPhone X @3x)
|
||
对于Android/Desktop: 这是逻辑分辨率
|
||
scale: iOS scale factor (2.0 for @2x, 3.0 for @3x),其他平台使用1.0
|
||
model_name: 使用的LLM模型名称
|
||
absolute_mode: 是否使用绝对坐标模式,如果为None则自动判断
|
||
"""
|
||
self.platform = platform.lower()
|
||
self.resolution = resolution
|
||
self.scale = scale
|
||
self.model_name = CURRENT_MODEL_NAME
|
||
self.llm = LLMClient(model_name=self.model_name)
|
||
|
||
|
||
# 确定坐标模式
|
||
if absolute_mode is None:
|
||
# iOS推荐使用绝对坐标,其他平台根据模型决定
|
||
if self.platform == "ios":
|
||
self.absolute_mode = True
|
||
else:
|
||
self.absolute_mode = is_absolute_coord_model(self.model_name)
|
||
else:
|
||
self.absolute_mode = absolute_mode
|
||
|
||
# 是否在截图上绘制网格(帮助LLM定位)
|
||
self.enable_grid = True
|
||
|
||
# 日志文件路径和初始化
|
||
self.log_dir = Path(os.environ.get('GUIAGENT_LOG_DIR', '/tmp/guiagent_logs'))
|
||
self.log_file_path = None
|
||
self._init_log()
|
||
|
||
# 生成系统提示词
|
||
self.system_prompt = self._generate_system_prompt()
|
||
|
||
logger.info(
|
||
f"Initialized GuiAgentDecisionMaker: platform={platform}, "
|
||
f"resolution={resolution}, scale={scale}, absolute_mode={self.absolute_mode}"
|
||
)
|
||
|
||
def _init_log(self) -> None:
|
||
"""初始化日志文件(追加模式,每次会话追加记录)"""
|
||
try:
|
||
self.log_dir.mkdir(parents=True, exist_ok=True)
|
||
self.log_file_path = self.log_dir / f"agent_{time.strftime('%Y%m%d')}.log"
|
||
with open(self.log_file_path, "a", encoding="utf-8") as f:
|
||
f.write(f"\n{'='*60}\n")
|
||
f.write(f"=== Agent Session: {time.strftime('%Y-%m-%d %H:%M:%S')} ===\n")
|
||
f.write(f"=== Platform: {self.platform}, Resolution: {self.resolution} ===\n")
|
||
f.write(f"{'='*60}\n")
|
||
logger.debug(f"日志文件初始化: {self.log_file_path}")
|
||
except Exception as e:
|
||
logger.error(f"日志文件初始化失败: {e}")
|
||
self.log_file_path = None
|
||
|
||
def _log(self, step: int, messages: list, response: str, tokens: Dict = None, error_type: str = None) -> None:
|
||
"""
|
||
记录到日志文件,包含发送给模型的上下文(去掉 base64 图片内容)
|
||
|
||
Args:
|
||
step: 当前步骤数
|
||
messages: 发送给LLM的消息列表
|
||
response: LLM响应文本
|
||
tokens: Token使用统计
|
||
error_type: 错误类型(如果有)
|
||
"""
|
||
if not self.log_file_path:
|
||
return
|
||
|
||
# 复制一份消息并脱敏 image base64,避免日志体积暴涨
|
||
def _sanitize_messages(msgs: list) -> list:
|
||
sanitized = []
|
||
for msg in msgs:
|
||
msg_copy = json.loads(json.dumps(msg)) # 简单深拷贝
|
||
content = msg_copy.get("content")
|
||
if isinstance(content, list):
|
||
for item in content:
|
||
if isinstance(item, dict) and item.get("type") == "image_url":
|
||
if "image_url" in item:
|
||
item["image_url"]["url"] = "<image_base64_omitted>"
|
||
msg_copy["content"] = content
|
||
sanitized.append(msg_copy)
|
||
return sanitized
|
||
|
||
try:
|
||
with open(self.log_file_path, "a", encoding="utf-8") as f:
|
||
f.write(f"\n=== Round {step} ===\n")
|
||
if error_type:
|
||
f.write(f"Error Type: {error_type}\n")
|
||
f.write("Messages Sent:\n")
|
||
f.write(json.dumps(_sanitize_messages(messages), ensure_ascii=False, indent=2))
|
||
f.write("\nResponse:\n")
|
||
f.write(f"{response}\n")
|
||
if tokens:
|
||
f.write(f"Tokens: in={tokens.get('input_tokens', 0)}, "
|
||
f"out={tokens.get('output_tokens', 0)}\n")
|
||
except Exception as e:
|
||
logger.error(f"写入日志文件失败: {e}")
|
||
|
||
def _generate_system_prompt(self) -> str:
|
||
"""生成平台特定的系统提示词"""
|
||
if self.platform == "ios":
|
||
return get_ios_prompt(
|
||
resolution=self.resolution,
|
||
scale=self.scale,
|
||
absolute_mode=self.absolute_mode
|
||
)
|
||
elif self.platform == "android":
|
||
return get_android_prompt(
|
||
resolution=self.resolution,
|
||
absolute_mode=self.absolute_mode
|
||
)
|
||
elif self.platform == "desktop":
|
||
return get_desktop_prompt(
|
||
resolution=self.resolution,
|
||
absolute_mode=self.absolute_mode
|
||
)
|
||
else:
|
||
raise ValueError(f"Unsupported platform: {self.platform}")
|
||
|
||
def decide_next_action(
|
||
self,
|
||
screenshot_path: str = None,
|
||
screenshot_base64: str = None,
|
||
width: int = None,
|
||
height: int = None,
|
||
context: ContextManager = None,
|
||
step: int = 0,
|
||
last_action_coords: tuple = None,
|
||
draw_grid_and_marker: bool = True
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
基于截图决策下一步动作
|
||
|
||
Args:
|
||
screenshot_path: 截图文件路径(与screenshot_base64二选一)
|
||
screenshot_base64: 截图的base64编码(与screenshot_path二选一)
|
||
width: 截图宽度(如果不提供则从图像获取)
|
||
height: 截图高度(如果不提供则从图像获取)
|
||
context: 上下文管理器(可选,用于多轮对话)
|
||
step: 当前步骤数(用于日志记录)
|
||
last_action_coords: 上次动作坐标,用于绘制标记
|
||
draw_grid_and_marker: 是否绘制网格和上次动作标记(卡住检测、场景验证等不依赖坐标的场景可关闭)
|
||
|
||
Returns:
|
||
{
|
||
"action_type": str, # 动作类型 (tap, drag, type, finished, etc.)
|
||
"action_data": Dict, # 动作参数(已转换为执行器格式)
|
||
"raw_response": str, # 原始LLM响应
|
||
"thought": str, # LLM的思考过程
|
||
"tokens": Dict, # Token使用统计
|
||
"is_finished": bool, # 任务是否完成
|
||
"success": bool # 决策是否成功
|
||
}
|
||
"""
|
||
messages = []
|
||
response = ""
|
||
tokens = {}
|
||
|
||
try:
|
||
# 1. 准备截图数据
|
||
if screenshot_base64 is None and screenshot_path:
|
||
screenshot_base64, width, height = self._load_and_process_screenshot(
|
||
screenshot_path,
|
||
last_action_coords=last_action_coords,
|
||
draw_grid_and_marker=draw_grid_and_marker
|
||
)
|
||
elif screenshot_base64:
|
||
# 如果提供了base64,也需要处理网格
|
||
if draw_grid_and_marker:
|
||
screenshot_base64 = self._add_grid_to_base64(screenshot_base64)
|
||
|
||
if not screenshot_base64:
|
||
self._log(step, [], "", error_type="screenshot_load_failed")
|
||
return self._error_result("No screenshot provided", error_type="screenshot_load_failed")
|
||
|
||
# 获取图像尺寸
|
||
if width is None or height is None:
|
||
img_width, img_height = self._get_image_size_from_base64(screenshot_base64)
|
||
width = width or img_width
|
||
height = height or img_height
|
||
|
||
if not width or not height:
|
||
self._log(step, [], "", error_type="resolution_failed")
|
||
return self._error_result("Failed to get image size", error_type="resolution_failed")
|
||
|
||
# 2. 准备上下文
|
||
if context is None:
|
||
context = ContextManager(system_prompt=self.system_prompt)
|
||
|
||
# 清理base64前缀
|
||
clean_base64 = strip_base64_prefix(screenshot_base64)
|
||
|
||
# 3. 添加截图到上下文
|
||
context.add_screenshot(clean_base64, width, height)
|
||
|
||
# 4. 构建消息并调用LLM
|
||
messages = context.build_messages()
|
||
|
||
logger.debug(f"[Step {step}] Calling LLM with {len(messages)} messages")
|
||
try:
|
||
response, _, tokens = self.llm.query(messages)
|
||
except Exception as e:
|
||
logger.error(f"[Step {step}] LLM调用失败: {e}")
|
||
self._log(step, messages, f"LLM Error: {e}", error_type="llm_call_failed")
|
||
return self._error_result(f"LLM call failed: {e}", error_type="llm_call_failed")
|
||
|
||
if not response:
|
||
self._log(step, messages, "", error_type="empty_response")
|
||
return self._error_result("Empty LLM response", error_type="empty_response")
|
||
|
||
# 5. 解析动作
|
||
parsed = parse_uitars_action(response)
|
||
action_type = parsed.get('action_type', '')
|
||
|
||
if not action_type:
|
||
logger.warning(f"[Step {step}] Failed to parse action from response: {response[:200]}")
|
||
self._log(step, messages, response, tokens, error_type="action_parse_failed")
|
||
return self._error_result("Failed to parse action", raw_response=response, error_type="action_parse_failed")
|
||
|
||
# 6. 转换为执行器格式
|
||
executor_action = convert_to_executor_action(parsed)
|
||
|
||
# 7. 记录成功的日志
|
||
self._log(step, messages, response, tokens)
|
||
|
||
# 8. 添加响应到上下文(为下一轮决策准备)
|
||
context.add_response(response)
|
||
|
||
# 增加全局步数计数
|
||
GuiAgentDecisionMaker.total_steps += 1
|
||
|
||
logger.info(f"[Step {step}] 决策成功: {action_type}")
|
||
|
||
# 9. 返回决策结果
|
||
return {
|
||
"action_type": action_type,
|
||
"action_data": executor_action,
|
||
"raw_response": response,
|
||
"thought": parsed.get('thought', ''),
|
||
"tokens": tokens or {},
|
||
"is_finished": action_type in ('finished', 'report_stuck_reason'),
|
||
"success": True
|
||
}
|
||
|
||
except KeyboardInterrupt:
|
||
raise
|
||
except Exception as e:
|
||
logger.exception(f"[Step {step}] Error in decide_next_action: {e}")
|
||
self._log(step, messages, response or str(e), tokens, error_type=f"exception:{type(e).__name__}")
|
||
# 返回错误结果而非抛出,让调用方决定如何处理
|
||
return self._error_result(f"Exception: {e}", error_type=f"exception:{type(e).__name__}")
|
||
|
||
def _load_and_process_screenshot(self, screenshot_path: str, last_action_coords: tuple = None, draw_grid_and_marker: bool = True) -> Tuple[Optional[str], int, int]:
|
||
"""从文件加载截图,添加网格和上次动作标记,并转换为base64"""
|
||
try:
|
||
img = Image.open(screenshot_path)
|
||
width, height = img.size
|
||
|
||
if draw_grid_and_marker:
|
||
# 添加网格(坐标标注与agent坐标系一致)
|
||
if self.enable_grid:
|
||
# 归一化模式标注0-1000,绝对模式标注实际像素
|
||
coord_range = None if self.absolute_mode else (1000, 1000)
|
||
img = draw_grid_on_image(img, coord_range=coord_range)
|
||
logger.debug(f"截图已添加网格: {screenshot_path}")
|
||
|
||
# 绘制上次动作坐标的绿圈标记
|
||
if last_action_coords:
|
||
x, y = last_action_coords
|
||
img = draw_last_action_marker(img, int(x), int(y))
|
||
logger.debug(f"截图已添加上次动作标记: ({x}, {y})")
|
||
|
||
# 转换为base64
|
||
img_base64 = image_to_base64(img)
|
||
return img_base64, width, height
|
||
except Exception as e:
|
||
logger.error(f"加载和处理截图失败 {screenshot_path}: {e}")
|
||
return None, 0, 0
|
||
|
||
def _add_grid_to_base64(self, screenshot_base64: str) -> str:
|
||
"""给base64截图添加网格"""
|
||
if not self.enable_grid:
|
||
return screenshot_base64
|
||
|
||
try:
|
||
import io
|
||
clean = strip_base64_prefix(screenshot_base64)
|
||
img_data = base64.b64decode(clean)
|
||
img = Image.open(io.BytesIO(img_data))
|
||
img = draw_grid_on_image(img, coord_range=None if self.absolute_mode else (1000, 1000))
|
||
return image_to_base64(img)
|
||
except Exception as e:
|
||
logger.error(f"添加网格失败: {e}")
|
||
return screenshot_base64
|
||
|
||
def _load_screenshot_base64(self, screenshot_path: str) -> Optional[str]:
|
||
"""从文件加载截图并转换为base64"""
|
||
try:
|
||
with open(screenshot_path, 'rb') as f:
|
||
img_data = f.read()
|
||
return base64.b64encode(img_data).decode('utf-8')
|
||
except Exception as e:
|
||
logger.error(f"Failed to load screenshot from {screenshot_path}: {e}")
|
||
return None
|
||
|
||
def _get_image_size_from_base64(self, base64_str: str) -> Tuple[int, int]:
|
||
"""从base64字符串获取图像尺寸"""
|
||
try:
|
||
import io
|
||
clean = strip_base64_prefix(base64_str)
|
||
img_data = base64.b64decode(clean)
|
||
img = Image.open(io.BytesIO(img_data))
|
||
return img.size
|
||
except Exception as e:
|
||
logger.error(f"Failed to get image size: {e}")
|
||
return 0, 0
|
||
|
||
def _error_result(self, error_msg: str, raw_response: str = "", error_type: str = "unknown") -> Dict[str, Any]:
|
||
"""生成错误结果,包含错误分类用于统计"""
|
||
return {
|
||
"action_type": "",
|
||
"action_data": {},
|
||
"raw_response": raw_response,
|
||
"thought": "",
|
||
"tokens": {},
|
||
"is_finished": False,
|
||
"success": False,
|
||
"error": error_msg,
|
||
"error_type": error_type
|
||
}
|
||
|
||
def verify_screen(
|
||
self,
|
||
category: str,
|
||
screenshot_path: str = None,
|
||
screenshot_base64: str = None,
|
||
get_screenshot_func=None
|
||
) -> bool:
|
||
"""
|
||
验证当前界面是否确实属于目标场景
|
||
|
||
Args:
|
||
category: 目标场景类别 (login, payment, etc.)
|
||
screenshot_path: 截图文件路径
|
||
screenshot_base64: 截图的base64编码
|
||
get_screenshot_func: 获取截图的回调函数,返回 (path, base64) 或 base64 字符串
|
||
|
||
Returns:
|
||
如果验证通过返回 True,否则返回 False
|
||
"""
|
||
logger.info(f"[验证] 正在验证当前界面是否为: {category}")
|
||
|
||
try:
|
||
# 获取截图
|
||
if screenshot_base64 is None:
|
||
if screenshot_path:
|
||
screenshot_base64 = self._load_screenshot_base64(screenshot_path)
|
||
elif get_screenshot_func:
|
||
result = get_screenshot_func()
|
||
if isinstance(result, tuple):
|
||
screenshot_path, screenshot_base64 = result
|
||
else:
|
||
screenshot_base64 = result
|
||
if screenshot_path and not screenshot_base64:
|
||
screenshot_base64 = self._load_screenshot_base64(screenshot_path)
|
||
|
||
if not screenshot_base64:
|
||
logger.warning("[验证] 无法获取截图,默认通过验证")
|
||
return True
|
||
|
||
clean_base64 = strip_base64_prefix(screenshot_base64)
|
||
|
||
# 构建验证提示词
|
||
verification_prompt = get_verification_prompt(category)
|
||
|
||
# 构建单次对话消息
|
||
messages = [
|
||
{
|
||
"role": "user",
|
||
"content": [
|
||
{"type": "text", "text": verification_prompt},
|
||
{
|
||
"type": "image_url",
|
||
"image_url": {"url": f"data:image/png;base64,{clean_base64}"}
|
||
}
|
||
]
|
||
}
|
||
]
|
||
|
||
# 调用模型
|
||
response, _, token_usage = self.llm.query(messages)
|
||
|
||
if not response:
|
||
logger.warning("[验证] 模型未返回响应,默认通过验证")
|
||
return True
|
||
|
||
logger.info(f"[验证响应]\n{response},Input Tokens: {token_usage.get('input_tokens', 0)},Output Tokens: {token_usage.get('output_tokens', 0)}")
|
||
|
||
# 解析结果
|
||
match = re.search(r"Result:\s*(YES|NO)", response, re.IGNORECASE)
|
||
if match:
|
||
result = match.group(1).upper()
|
||
if result == "YES":
|
||
logger.info(f"[验证成功] 确认当前为 {category} 场景")
|
||
return True
|
||
else:
|
||
logger.info(f"[验证失败] 当前界面不符合 {category} 场景描述")
|
||
return False
|
||
|
||
# 如果没找到标准格式,简单检查关键词
|
||
if "YES" in response.upper() and "NO" not in response.upper():
|
||
return True
|
||
|
||
return False
|
||
|
||
except Exception as e:
|
||
logger.error(f"[验证] 验证过程中发生异常: {e}")
|
||
return True # 发生异常时默认通过,避免中断流程
|
||
|
||
def create_context(self, instruction: str = None) -> ContextManager:
|
||
"""
|
||
创建新的上下文管理器
|
||
|
||
Args:
|
||
instruction: 初始任务指令(可选)
|
||
|
||
Returns:
|
||
ContextManager实例
|
||
"""
|
||
context = ContextManager(system_prompt=self.system_prompt)
|
||
if instruction:
|
||
context.add_instruction(instruction)
|
||
return context
|
||
|
||
|
||
if __name__ == "__main__":
|
||
"""
|
||
模块化测试和调试入口
|
||
|
||
用于单独测试和调试 GuiAgent 的决策功能,无需启动完整的自动化框架。
|
||
直接修改下面的配置参数即可进行测试。
|
||
|
||
使用示例:
|
||
python -m DroidBot.guiagent_core.decision_maker
|
||
或
|
||
python DroidBot/guiagent_core/decision_maker.py
|
||
"""
|
||
import sys
|
||
|
||
# ==================== 配置参数(根据需要修改) ====================
|
||
|
||
# === 截图模式配置 ===
|
||
# 设置为 True 可从连接的设备实时捕获截图,False 则使用指定的截图文件
|
||
USE_DEVICE_SCREENSHOT = True
|
||
|
||
# 截图文件路径(当 USE_DEVICE_SCREENSHOT=False 时使用)
|
||
SCREENSHOT_PATH = "/path/to/screenshot.png"
|
||
|
||
# === 设备配置(当 USE_DEVICE_SCREENSHOT=True 时使用) ===
|
||
# Android 设备序列号(None 表示使用默认连接的设备)
|
||
DEVICE_SERIAL = None
|
||
|
||
# iOS WDA 服务地址
|
||
WDA_URL = ""
|
||
|
||
# 设备截图保存目录
|
||
DEVICE_OUTPUT_DIR = "./output/guiagent_debug"
|
||
|
||
# === 任务配置 ===
|
||
# 任务指令
|
||
INSTRUCTION = "探索当前界面并执行合理操作"
|
||
|
||
# 平台类型: 'android', 'ios', 'desktop'
|
||
PLATFORM = "ios"
|
||
|
||
# 屏幕分辨率 (width, height),None 表示自动从截图获取
|
||
RESOLUTION = None # 例如: (1080, 2340)
|
||
|
||
# iOS scale factor (2.0 for @2x, 3.0 for @3x)
|
||
SCALE = 1.0
|
||
|
||
# 最大执行步数(用于多轮测试)
|
||
MAX_STEPS = 1
|
||
|
||
# 是否强制使用绝对坐标模式
|
||
ABSOLUTE_MODE = None # None 表示自动判断
|
||
|
||
# 是否禁用截图网格线
|
||
DISABLE_GRID = False
|
||
|
||
# 验证模式:验证当前界面是否属于指定场景(None 表示不验证)
|
||
# 例如: "login", "payment" 等
|
||
VERIFY_CATEGORY = None
|
||
|
||
# 是否启用调试模式(显示详细日志和原始响应)
|
||
DEBUG_MODE = False
|
||
|
||
# ================================================================
|
||
|
||
# 配置日志
|
||
logging.basicConfig(
|
||
level=logging.DEBUG if DEBUG_MODE else logging.INFO,
|
||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||
)
|
||
|
||
if DEBUG_MODE:
|
||
logger.setLevel(logging.DEBUG)
|
||
|
||
# 初始化设备(如果使用设备截图模式)
|
||
device = None
|
||
if USE_DEVICE_SCREENSHOT:
|
||
logger.info(f"\n{'='*60}")
|
||
logger.info(f"设备截图模式已启用")
|
||
logger.info(f"{'='*60}\n")
|
||
|
||
try:
|
||
if PLATFORM == "android":
|
||
logger.info("正在连接 Android 设备...")
|
||
try:
|
||
from DroidBot.platforms.android import AndroidDevice
|
||
except ImportError:
|
||
from platforms.android import AndroidDevice
|
||
|
||
device = AndroidDevice(
|
||
device_serial=DEVICE_SERIAL,
|
||
output_dir=DEVICE_OUTPUT_DIR
|
||
)
|
||
device.set_up()
|
||
device.connect()
|
||
logger.info(f"✓ Android 设备已连接: {device.device_serial}")
|
||
|
||
elif PLATFORM == "ios":
|
||
logger.info("正在连接 iOS 设备...")
|
||
try:
|
||
from DroidBot.platforms.ios import IOSDevice
|
||
except ImportError:
|
||
from platforms.ios import IOSDevice
|
||
|
||
device = IOSDevice(
|
||
wda_url=WDA_URL,
|
||
output_dir=DEVICE_OUTPUT_DIR
|
||
)
|
||
device.set_up()
|
||
device.connect()
|
||
logger.info(f"✓ iOS 设备已连接 (WDA: {WDA_URL})")
|
||
|
||
else:
|
||
logger.error(f"设备截图模式不支持平台: {PLATFORM}")
|
||
sys.exit(1)
|
||
|
||
# 从设备捕获截图
|
||
logger.info("正在从设备捕获截图...")
|
||
SCREENSHOT_PATH = device.take_screenshot()
|
||
|
||
if not SCREENSHOT_PATH:
|
||
logger.error("从设备捕获截图失败")
|
||
sys.exit(1)
|
||
|
||
logger.info(f"✓ 截图已保存: {SCREENSHOT_PATH}")
|
||
|
||
except Exception as e:
|
||
logger.error(f"设备连接或截图失败: {e}")
|
||
if device:
|
||
try:
|
||
device.disconnect()
|
||
except:
|
||
pass
|
||
sys.exit(1)
|
||
else:
|
||
# 文件模式:检查截图文件是否存在
|
||
if not os.path.exists(SCREENSHOT_PATH):
|
||
logger.error(f"截图文件不存在: {SCREENSHOT_PATH}")
|
||
logger.error(f"请修改 SCREENSHOT_PATH 参数为有效的截图文件路径")
|
||
logger.error(f"或者设置 USE_DEVICE_SCREENSHOT = True 从设备捕获截图")
|
||
sys.exit(1)
|
||
|
||
# 创建决策引擎
|
||
logger.info(f"\n{'='*60}")
|
||
logger.info(f"初始化 GuiAgent 决策引擎")
|
||
logger.info(f"{'='*60}")
|
||
logger.info(f"平台: {PLATFORM}, 分辨率: {RESOLUTION}, Scale: {SCALE}")
|
||
|
||
try:
|
||
decision_maker = GuiAgentDecisionMaker(
|
||
platform=PLATFORM,
|
||
resolution=RESOLUTION,
|
||
scale=SCALE,
|
||
absolute_mode=ABSOLUTE_MODE
|
||
)
|
||
|
||
# 配置网格
|
||
if DISABLE_GRID:
|
||
decision_maker.enable_grid = False
|
||
logger.info("已禁用截图网格")
|
||
|
||
# 验证模式
|
||
if VERIFY_CATEGORY:
|
||
logger.info(f"\n{'='*60}")
|
||
logger.info(f"验证模式: 检查当前界面是否为 {VERIFY_CATEGORY} 场景")
|
||
logger.info(f"{'='*60}\n")
|
||
|
||
is_valid = decision_maker.verify_screen(
|
||
category=VERIFY_CATEGORY,
|
||
screenshot_path=SCREENSHOT_PATH
|
||
)
|
||
|
||
print(f"\n{'='*60}")
|
||
print(f"验证结果: {'✓ 通过' if is_valid else '✗ 不通过'}")
|
||
print(f"{'='*60}\n")
|
||
|
||
sys.exit(0 if is_valid else 1)
|
||
|
||
# 决策模式
|
||
logger.info(f"\n{'='*60}")
|
||
logger.info(f"任务指令: {INSTRUCTION}")
|
||
logger.info(f"截图文件: {SCREENSHOT_PATH}")
|
||
logger.info(f"最大步数: {MAX_STEPS}")
|
||
logger.info(f"{'='*60}\n")
|
||
|
||
# 创建上下文
|
||
context = decision_maker.create_context(instruction=INSTRUCTION)
|
||
|
||
# 多轮决策测试
|
||
for step in range(MAX_STEPS):
|
||
logger.info(f"\n{'='*60}")
|
||
logger.info(f"第 {step + 1} 步决策")
|
||
logger.info(f"{'='*60}\n")
|
||
|
||
# 执行决策
|
||
result = decision_maker.decide_next_action(
|
||
screenshot_path=SCREENSHOT_PATH,
|
||
context=context,
|
||
step=step + 1
|
||
)
|
||
|
||
# 打印结果
|
||
print(f"\n{'='*60}")
|
||
print(f"决策结果 (步骤 {step + 1})")
|
||
print(f"{'='*60}")
|
||
print(f"成功: {result.get('success')}")
|
||
print(f"动作类型: {result.get('action_type')}")
|
||
print(f"是否完成: {result.get('is_finished')}")
|
||
|
||
if result.get('thought'):
|
||
print(f"\n思考过程:")
|
||
print(f" {result.get('thought')}")
|
||
|
||
if result.get('action_data'):
|
||
print(f"\n动作数据:")
|
||
for key, value in result.get('action_data', {}).items():
|
||
print(f" {key}: {value}")
|
||
|
||
if result.get('tokens'):
|
||
tokens = result.get('tokens')
|
||
print(f"\nToken 使用:")
|
||
print(f" 输入: {tokens.get('input_tokens', 0)}")
|
||
print(f" 输出: {tokens.get('output_tokens', 0)}")
|
||
print(f" 总计: {tokens.get('total', 0)}")
|
||
|
||
if result.get('error'):
|
||
print(f"\n错误信息: {result.get('error')}")
|
||
print(f"错误类型: {result.get('error_type')}")
|
||
|
||
if DEBUG_MODE and result.get('raw_response'):
|
||
print(f"\n原始响应:")
|
||
print(f"{result.get('raw_response')}")
|
||
|
||
print(f"{'='*60}\n")
|
||
|
||
# 检查是否完成或失败
|
||
if result.get('is_finished'):
|
||
logger.info(f"✓ 任务完成!")
|
||
break
|
||
|
||
if not result.get('success'):
|
||
logger.error(f"✗ 决策失败,停止执行")
|
||
sys.exit(1)
|
||
|
||
# 如果是多步测试,模拟等待
|
||
if step < MAX_STEPS - 1:
|
||
logger.info("等待 2 秒后继续下一步...")
|
||
time.sleep(2)
|
||
|
||
logger.info(f"\n测试完成!日志文件: {decision_maker.log_file_path}")
|
||
|
||
except KeyboardInterrupt:
|
||
logger.info("\n用户中断测试")
|
||
sys.exit(130)
|
||
except Exception as e:
|
||
logger.exception(f"测试过程中发生异常: {e}")
|
||
sys.exit(1)
|
||
finally:
|
||
# 清理设备连接
|
||
if device:
|
||
logger.info("\n正在断开设备连接...")
|
||
try:
|
||
device.disconnect()
|
||
logger.info("✓ 设备已断开")
|
||
except Exception as e:
|
||
logger.warning(f"设备断开时出现警告: {e}")
|
||
|