autool/DroidBot/guiagent_core/decision_maker.py
2026-06-17 19:44:18 +08:00

767 lines
30 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""
Decision Maker - Core GuiAgent decision engine without device coupling
Extracted from GuiAgent/core/agent.py, this module provides stateless LLM-based
decision making for GUI automation without managing device connections.
"""
import base64
import logging
import json
import time
import os
from pathlib import Path
from typing import Dict, Any, Tuple, Optional
from PIL import Image
import sys
# 智能导入:先尝试相对导入,失败则使用绝对导入
# 支持:模块导入、直接运行、调试器运行等多种场景
try:
from .context_manager import ContextManager
from .llm_client import LLMClient
from .prompt_builder import get_ios_prompt, get_android_prompt, get_desktop_prompt, get_verification_prompt
from .utils import parse_uitars_action, convert_to_executor_action, strip_base64_prefix, draw_grid_on_image, draw_last_action_marker, image_to_base64
from .constants import is_absolute_coord_model, CURRENT_MODEL_NAME
except ImportError:
# 相对导入失败,添加项目根目录到 sys.path 并使用绝对导入
current_file = Path(__file__).resolve()
project_root = current_file.parent.parent.parent
if str(project_root) not in sys.path:
sys.path.insert(0, str(project_root))
from DroidBot.guiagent_core.context_manager import ContextManager
from DroidBot.guiagent_core.llm_client import LLMClient
from DroidBot.guiagent_core.prompt_builder import get_ios_prompt, get_android_prompt, get_desktop_prompt, get_verification_prompt
from DroidBot.guiagent_core.utils import parse_uitars_action, convert_to_executor_action, strip_base64_prefix, draw_grid_on_image, draw_last_action_marker, image_to_base64
from DroidBot.guiagent_core.constants import is_absolute_coord_model, CURRENT_MODEL_NAME
import re
logger = logging.getLogger(__name__)
class GuiAgentDecisionMaker:
"""
GuiAgent 核心决策引擎(无状态,不管理设备)
提供基于LLM的GUI自动化决策能力支持多平台iOS, Android, Desktop
专为DroidBot等自动化框架设计实现决策与执行的解耦。
"""
# 全局步数计数器用于记录本轮采集时的GUI Agent步数总和
total_steps = 0
def __init__(
self,
platform: str,
resolution: Tuple[int, int] = None,
scale: float = 1.0,
absolute_mode: bool = None
):
"""
初始化决策引擎
Args:
platform: 平台类型 ("ios", "android", "desktop")
resolution: 屏幕分辨率 (width, height)
对于iOS: 这是截图的物理像素尺寸 (e.g., 1125x2436 for iPhone X @3x)
对于Android/Desktop: 这是逻辑分辨率
scale: iOS scale factor (2.0 for @2x, 3.0 for @3x)其他平台使用1.0
model_name: 使用的LLM模型名称
absolute_mode: 是否使用绝对坐标模式如果为None则自动判断
"""
self.platform = platform.lower()
self.resolution = resolution
self.scale = scale
self.model_name = CURRENT_MODEL_NAME
self.llm = LLMClient(model_name=self.model_name)
# 确定坐标模式
if absolute_mode is None:
# iOS推荐使用绝对坐标其他平台根据模型决定
if self.platform == "ios":
self.absolute_mode = True
else:
self.absolute_mode = is_absolute_coord_model(self.model_name)
else:
self.absolute_mode = absolute_mode
# 是否在截图上绘制网格帮助LLM定位
self.enable_grid = True
# 日志文件路径和初始化
self.log_dir = Path(os.environ.get('GUIAGENT_LOG_DIR', '/tmp/guiagent_logs'))
self.log_file_path = None
self._init_log()
# 生成系统提示词
self.system_prompt = self._generate_system_prompt()
logger.info(
f"Initialized GuiAgentDecisionMaker: platform={platform}, "
f"resolution={resolution}, scale={scale}, absolute_mode={self.absolute_mode}"
)
def _init_log(self) -> None:
"""初始化日志文件(追加模式,每次会话追加记录)"""
try:
self.log_dir.mkdir(parents=True, exist_ok=True)
self.log_file_path = self.log_dir / f"agent_{time.strftime('%Y%m%d')}.log"
with open(self.log_file_path, "a", encoding="utf-8") as f:
f.write(f"\n{'='*60}\n")
f.write(f"=== Agent Session: {time.strftime('%Y-%m-%d %H:%M:%S')} ===\n")
f.write(f"=== Platform: {self.platform}, Resolution: {self.resolution} ===\n")
f.write(f"{'='*60}\n")
logger.debug(f"日志文件初始化: {self.log_file_path}")
except Exception as e:
logger.error(f"日志文件初始化失败: {e}")
self.log_file_path = None
def _log(self, step: int, messages: list, response: str, tokens: Dict = None, error_type: str = None) -> None:
"""
记录到日志文件,包含发送给模型的上下文(去掉 base64 图片内容)
Args:
step: 当前步骤数
messages: 发送给LLM的消息列表
response: LLM响应文本
tokens: Token使用统计
error_type: 错误类型(如果有)
"""
if not self.log_file_path:
return
# 复制一份消息并脱敏 image base64避免日志体积暴涨
def _sanitize_messages(msgs: list) -> list:
sanitized = []
for msg in msgs:
msg_copy = json.loads(json.dumps(msg)) # 简单深拷贝
content = msg_copy.get("content")
if isinstance(content, list):
for item in content:
if isinstance(item, dict) and item.get("type") == "image_url":
if "image_url" in item:
item["image_url"]["url"] = "<image_base64_omitted>"
msg_copy["content"] = content
sanitized.append(msg_copy)
return sanitized
try:
with open(self.log_file_path, "a", encoding="utf-8") as f:
f.write(f"\n=== Round {step} ===\n")
if error_type:
f.write(f"Error Type: {error_type}\n")
f.write("Messages Sent:\n")
f.write(json.dumps(_sanitize_messages(messages), ensure_ascii=False, indent=2))
f.write("\nResponse:\n")
f.write(f"{response}\n")
if tokens:
f.write(f"Tokens: in={tokens.get('input_tokens', 0)}, "
f"out={tokens.get('output_tokens', 0)}\n")
except Exception as e:
logger.error(f"写入日志文件失败: {e}")
def _generate_system_prompt(self) -> str:
"""生成平台特定的系统提示词"""
if self.platform == "ios":
return get_ios_prompt(
resolution=self.resolution,
scale=self.scale,
absolute_mode=self.absolute_mode
)
elif self.platform == "android":
return get_android_prompt(
resolution=self.resolution,
absolute_mode=self.absolute_mode
)
elif self.platform == "desktop":
return get_desktop_prompt(
resolution=self.resolution,
absolute_mode=self.absolute_mode
)
else:
raise ValueError(f"Unsupported platform: {self.platform}")
def decide_next_action(
self,
screenshot_path: str = None,
screenshot_base64: str = None,
width: int = None,
height: int = None,
context: ContextManager = None,
step: int = 0,
last_action_coords: tuple = None,
draw_grid_and_marker: bool = True
) -> Dict[str, Any]:
"""
基于截图决策下一步动作
Args:
screenshot_path: 截图文件路径与screenshot_base64二选一
screenshot_base64: 截图的base64编码与screenshot_path二选一
width: 截图宽度(如果不提供则从图像获取)
height: 截图高度(如果不提供则从图像获取)
context: 上下文管理器(可选,用于多轮对话)
step: 当前步骤数(用于日志记录)
last_action_coords: 上次动作坐标,用于绘制标记
draw_grid_and_marker: 是否绘制网格和上次动作标记(卡住检测、场景验证等不依赖坐标的场景可关闭)
Returns:
{
"action_type": str, # 动作类型 (tap, drag, type, finished, etc.)
"action_data": Dict, # 动作参数(已转换为执行器格式)
"raw_response": str, # 原始LLM响应
"thought": str, # LLM的思考过程
"tokens": Dict, # Token使用统计
"is_finished": bool, # 任务是否完成
"success": bool # 决策是否成功
}
"""
messages = []
response = ""
tokens = {}
try:
# 1. 准备截图数据
if screenshot_base64 is None and screenshot_path:
screenshot_base64, width, height = self._load_and_process_screenshot(
screenshot_path,
last_action_coords=last_action_coords,
draw_grid_and_marker=draw_grid_and_marker
)
elif screenshot_base64:
# 如果提供了base64也需要处理网格
if draw_grid_and_marker:
screenshot_base64 = self._add_grid_to_base64(screenshot_base64)
if not screenshot_base64:
self._log(step, [], "", error_type="screenshot_load_failed")
return self._error_result("No screenshot provided", error_type="screenshot_load_failed")
# 获取图像尺寸
if width is None or height is None:
img_width, img_height = self._get_image_size_from_base64(screenshot_base64)
width = width or img_width
height = height or img_height
if not width or not height:
self._log(step, [], "", error_type="resolution_failed")
return self._error_result("Failed to get image size", error_type="resolution_failed")
# 2. 准备上下文
if context is None:
context = ContextManager(system_prompt=self.system_prompt)
# 清理base64前缀
clean_base64 = strip_base64_prefix(screenshot_base64)
# 3. 添加截图到上下文
context.add_screenshot(clean_base64, width, height)
# 4. 构建消息并调用LLM
messages = context.build_messages()
logger.debug(f"[Step {step}] Calling LLM with {len(messages)} messages")
try:
response, _, tokens = self.llm.query(messages)
except Exception as e:
logger.error(f"[Step {step}] LLM调用失败: {e}")
self._log(step, messages, f"LLM Error: {e}", error_type="llm_call_failed")
return self._error_result(f"LLM call failed: {e}", error_type="llm_call_failed")
if not response:
self._log(step, messages, "", error_type="empty_response")
return self._error_result("Empty LLM response", error_type="empty_response")
# 5. 解析动作
parsed = parse_uitars_action(response)
action_type = parsed.get('action_type', '')
if not action_type:
logger.warning(f"[Step {step}] Failed to parse action from response: {response[:200]}")
self._log(step, messages, response, tokens, error_type="action_parse_failed")
return self._error_result("Failed to parse action", raw_response=response, error_type="action_parse_failed")
# 6. 转换为执行器格式
executor_action = convert_to_executor_action(parsed)
# 7. 记录成功的日志
self._log(step, messages, response, tokens)
# 8. 添加响应到上下文(为下一轮决策准备)
context.add_response(response)
# 增加全局步数计数
GuiAgentDecisionMaker.total_steps += 1
logger.info(f"[Step {step}] 决策成功: {action_type}")
# 9. 返回决策结果
return {
"action_type": action_type,
"action_data": executor_action,
"raw_response": response,
"thought": parsed.get('thought', ''),
"tokens": tokens or {},
"is_finished": action_type in ('finished', 'report_stuck_reason'),
"success": True
}
except KeyboardInterrupt:
raise
except Exception as e:
logger.exception(f"[Step {step}] Error in decide_next_action: {e}")
self._log(step, messages, response or str(e), tokens, error_type=f"exception:{type(e).__name__}")
# 返回错误结果而非抛出,让调用方决定如何处理
return self._error_result(f"Exception: {e}", error_type=f"exception:{type(e).__name__}")
def _load_and_process_screenshot(self, screenshot_path: str, last_action_coords: tuple = None, draw_grid_and_marker: bool = True) -> Tuple[Optional[str], int, int]:
"""从文件加载截图添加网格和上次动作标记并转换为base64"""
try:
img = Image.open(screenshot_path)
width, height = img.size
if draw_grid_and_marker:
# 添加网格坐标标注与agent坐标系一致
if self.enable_grid:
# 归一化模式标注0-1000绝对模式标注实际像素
coord_range = None if self.absolute_mode else (1000, 1000)
img = draw_grid_on_image(img, coord_range=coord_range)
logger.debug(f"截图已添加网格: {screenshot_path}")
# 绘制上次动作坐标的绿圈标记
if last_action_coords:
x, y = last_action_coords
img = draw_last_action_marker(img, int(x), int(y))
logger.debug(f"截图已添加上次动作标记: ({x}, {y})")
# 转换为base64
img_base64 = image_to_base64(img)
return img_base64, width, height
except Exception as e:
logger.error(f"加载和处理截图失败 {screenshot_path}: {e}")
return None, 0, 0
def _add_grid_to_base64(self, screenshot_base64: str) -> str:
"""给base64截图添加网格"""
if not self.enable_grid:
return screenshot_base64
try:
import io
clean = strip_base64_prefix(screenshot_base64)
img_data = base64.b64decode(clean)
img = Image.open(io.BytesIO(img_data))
img = draw_grid_on_image(img, coord_range=None if self.absolute_mode else (1000, 1000))
return image_to_base64(img)
except Exception as e:
logger.error(f"添加网格失败: {e}")
return screenshot_base64
def _load_screenshot_base64(self, screenshot_path: str) -> Optional[str]:
"""从文件加载截图并转换为base64"""
try:
with open(screenshot_path, 'rb') as f:
img_data = f.read()
return base64.b64encode(img_data).decode('utf-8')
except Exception as e:
logger.error(f"Failed to load screenshot from {screenshot_path}: {e}")
return None
def _get_image_size_from_base64(self, base64_str: str) -> Tuple[int, int]:
"""从base64字符串获取图像尺寸"""
try:
import io
clean = strip_base64_prefix(base64_str)
img_data = base64.b64decode(clean)
img = Image.open(io.BytesIO(img_data))
return img.size
except Exception as e:
logger.error(f"Failed to get image size: {e}")
return 0, 0
def _error_result(self, error_msg: str, raw_response: str = "", error_type: str = "unknown") -> Dict[str, Any]:
"""生成错误结果,包含错误分类用于统计"""
return {
"action_type": "",
"action_data": {},
"raw_response": raw_response,
"thought": "",
"tokens": {},
"is_finished": False,
"success": False,
"error": error_msg,
"error_type": error_type
}
def verify_screen(
self,
category: str,
screenshot_path: str = None,
screenshot_base64: str = None,
get_screenshot_func=None
) -> bool:
"""
验证当前界面是否确实属于目标场景
Args:
category: 目标场景类别 (login, payment, etc.)
screenshot_path: 截图文件路径
screenshot_base64: 截图的base64编码
get_screenshot_func: 获取截图的回调函数,返回 (path, base64) 或 base64 字符串
Returns:
如果验证通过返回 True否则返回 False
"""
logger.info(f"[验证] 正在验证当前界面是否为: {category}")
try:
# 获取截图
if screenshot_base64 is None:
if screenshot_path:
screenshot_base64 = self._load_screenshot_base64(screenshot_path)
elif get_screenshot_func:
result = get_screenshot_func()
if isinstance(result, tuple):
screenshot_path, screenshot_base64 = result
else:
screenshot_base64 = result
if screenshot_path and not screenshot_base64:
screenshot_base64 = self._load_screenshot_base64(screenshot_path)
if not screenshot_base64:
logger.warning("[验证] 无法获取截图,默认通过验证")
return True
clean_base64 = strip_base64_prefix(screenshot_base64)
# 构建验证提示词
verification_prompt = get_verification_prompt(category)
# 构建单次对话消息
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": verification_prompt},
{
"type": "image_url",
"image_url": {"url": f"data:image/png;base64,{clean_base64}"}
}
]
}
]
# 调用模型
response, _, token_usage = self.llm.query(messages)
if not response:
logger.warning("[验证] 模型未返回响应,默认通过验证")
return True
logger.info(f"[验证响应]\n{response},Input Tokens: {token_usage.get('input_tokens', 0)},Output Tokens: {token_usage.get('output_tokens', 0)}")
# 解析结果
match = re.search(r"Result:\s*(YES|NO)", response, re.IGNORECASE)
if match:
result = match.group(1).upper()
if result == "YES":
logger.info(f"[验证成功] 确认当前为 {category} 场景")
return True
else:
logger.info(f"[验证失败] 当前界面不符合 {category} 场景描述")
return False
# 如果没找到标准格式,简单检查关键词
if "YES" in response.upper() and "NO" not in response.upper():
return True
return False
except Exception as e:
logger.error(f"[验证] 验证过程中发生异常: {e}")
return True # 发生异常时默认通过,避免中断流程
def create_context(self, instruction: str = None) -> ContextManager:
"""
创建新的上下文管理器
Args:
instruction: 初始任务指令(可选)
Returns:
ContextManager实例
"""
context = ContextManager(system_prompt=self.system_prompt)
if instruction:
context.add_instruction(instruction)
return context
if __name__ == "__main__":
"""
模块化测试和调试入口
用于单独测试和调试 GuiAgent 的决策功能,无需启动完整的自动化框架。
直接修改下面的配置参数即可进行测试。
使用示例:
python -m DroidBot.guiagent_core.decision_maker
python DroidBot/guiagent_core/decision_maker.py
"""
import sys
# ==================== 配置参数(根据需要修改) ====================
# === 截图模式配置 ===
# 设置为 True 可从连接的设备实时捕获截图False 则使用指定的截图文件
USE_DEVICE_SCREENSHOT = True
# 截图文件路径(当 USE_DEVICE_SCREENSHOT=False 时使用)
SCREENSHOT_PATH = "/path/to/screenshot.png"
# === 设备配置(当 USE_DEVICE_SCREENSHOT=True 时使用) ===
# Android 设备序列号None 表示使用默认连接的设备)
DEVICE_SERIAL = None
# iOS WDA 服务地址
WDA_URL = ""
# 设备截图保存目录
DEVICE_OUTPUT_DIR = "./output/guiagent_debug"
# === 任务配置 ===
# 任务指令
INSTRUCTION = "探索当前界面并执行合理操作"
# 平台类型: 'android', 'ios', 'desktop'
PLATFORM = "ios"
# 屏幕分辨率 (width, height)None 表示自动从截图获取
RESOLUTION = None # 例如: (1080, 2340)
# iOS scale factor (2.0 for @2x, 3.0 for @3x)
SCALE = 1.0
# 最大执行步数(用于多轮测试)
MAX_STEPS = 1
# 是否强制使用绝对坐标模式
ABSOLUTE_MODE = None # None 表示自动判断
# 是否禁用截图网格线
DISABLE_GRID = False
# 验证模式验证当前界面是否属于指定场景None 表示不验证)
# 例如: "login", "payment" 等
VERIFY_CATEGORY = None
# 是否启用调试模式(显示详细日志和原始响应)
DEBUG_MODE = False
# ================================================================
# 配置日志
logging.basicConfig(
level=logging.DEBUG if DEBUG_MODE else logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
if DEBUG_MODE:
logger.setLevel(logging.DEBUG)
# 初始化设备(如果使用设备截图模式)
device = None
if USE_DEVICE_SCREENSHOT:
logger.info(f"\n{'='*60}")
logger.info(f"设备截图模式已启用")
logger.info(f"{'='*60}\n")
try:
if PLATFORM == "android":
logger.info("正在连接 Android 设备...")
try:
from DroidBot.platforms.android import AndroidDevice
except ImportError:
from platforms.android import AndroidDevice
device = AndroidDevice(
device_serial=DEVICE_SERIAL,
output_dir=DEVICE_OUTPUT_DIR
)
device.set_up()
device.connect()
logger.info(f"✓ Android 设备已连接: {device.device_serial}")
elif PLATFORM == "ios":
logger.info("正在连接 iOS 设备...")
try:
from DroidBot.platforms.ios import IOSDevice
except ImportError:
from platforms.ios import IOSDevice
device = IOSDevice(
wda_url=WDA_URL,
output_dir=DEVICE_OUTPUT_DIR
)
device.set_up()
device.connect()
logger.info(f"✓ iOS 设备已连接 (WDA: {WDA_URL})")
else:
logger.error(f"设备截图模式不支持平台: {PLATFORM}")
sys.exit(1)
# 从设备捕获截图
logger.info("正在从设备捕获截图...")
SCREENSHOT_PATH = device.take_screenshot()
if not SCREENSHOT_PATH:
logger.error("从设备捕获截图失败")
sys.exit(1)
logger.info(f"✓ 截图已保存: {SCREENSHOT_PATH}")
except Exception as e:
logger.error(f"设备连接或截图失败: {e}")
if device:
try:
device.disconnect()
except:
pass
sys.exit(1)
else:
# 文件模式:检查截图文件是否存在
if not os.path.exists(SCREENSHOT_PATH):
logger.error(f"截图文件不存在: {SCREENSHOT_PATH}")
logger.error(f"请修改 SCREENSHOT_PATH 参数为有效的截图文件路径")
logger.error(f"或者设置 USE_DEVICE_SCREENSHOT = True 从设备捕获截图")
sys.exit(1)
# 创建决策引擎
logger.info(f"\n{'='*60}")
logger.info(f"初始化 GuiAgent 决策引擎")
logger.info(f"{'='*60}")
logger.info(f"平台: {PLATFORM}, 分辨率: {RESOLUTION}, Scale: {SCALE}")
try:
decision_maker = GuiAgentDecisionMaker(
platform=PLATFORM,
resolution=RESOLUTION,
scale=SCALE,
absolute_mode=ABSOLUTE_MODE
)
# 配置网格
if DISABLE_GRID:
decision_maker.enable_grid = False
logger.info("已禁用截图网格")
# 验证模式
if VERIFY_CATEGORY:
logger.info(f"\n{'='*60}")
logger.info(f"验证模式: 检查当前界面是否为 {VERIFY_CATEGORY} 场景")
logger.info(f"{'='*60}\n")
is_valid = decision_maker.verify_screen(
category=VERIFY_CATEGORY,
screenshot_path=SCREENSHOT_PATH
)
print(f"\n{'='*60}")
print(f"验证结果: {'✓ 通过' if is_valid else '✗ 不通过'}")
print(f"{'='*60}\n")
sys.exit(0 if is_valid else 1)
# 决策模式
logger.info(f"\n{'='*60}")
logger.info(f"任务指令: {INSTRUCTION}")
logger.info(f"截图文件: {SCREENSHOT_PATH}")
logger.info(f"最大步数: {MAX_STEPS}")
logger.info(f"{'='*60}\n")
# 创建上下文
context = decision_maker.create_context(instruction=INSTRUCTION)
# 多轮决策测试
for step in range(MAX_STEPS):
logger.info(f"\n{'='*60}")
logger.info(f"{step + 1} 步决策")
logger.info(f"{'='*60}\n")
# 执行决策
result = decision_maker.decide_next_action(
screenshot_path=SCREENSHOT_PATH,
context=context,
step=step + 1
)
# 打印结果
print(f"\n{'='*60}")
print(f"决策结果 (步骤 {step + 1})")
print(f"{'='*60}")
print(f"成功: {result.get('success')}")
print(f"动作类型: {result.get('action_type')}")
print(f"是否完成: {result.get('is_finished')}")
if result.get('thought'):
print(f"\n思考过程:")
print(f" {result.get('thought')}")
if result.get('action_data'):
print(f"\n动作数据:")
for key, value in result.get('action_data', {}).items():
print(f" {key}: {value}")
if result.get('tokens'):
tokens = result.get('tokens')
print(f"\nToken 使用:")
print(f" 输入: {tokens.get('input_tokens', 0)}")
print(f" 输出: {tokens.get('output_tokens', 0)}")
print(f" 总计: {tokens.get('total', 0)}")
if result.get('error'):
print(f"\n错误信息: {result.get('error')}")
print(f"错误类型: {result.get('error_type')}")
if DEBUG_MODE and result.get('raw_response'):
print(f"\n原始响应:")
print(f"{result.get('raw_response')}")
print(f"{'='*60}\n")
# 检查是否完成或失败
if result.get('is_finished'):
logger.info(f"✓ 任务完成!")
break
if not result.get('success'):
logger.error(f"✗ 决策失败,停止执行")
sys.exit(1)
# 如果是多步测试,模拟等待
if step < MAX_STEPS - 1:
logger.info("等待 2 秒后继续下一步...")
time.sleep(2)
logger.info(f"\n测试完成!日志文件: {decision_maker.log_file_path}")
except KeyboardInterrupt:
logger.info("\n用户中断测试")
sys.exit(130)
except Exception as e:
logger.exception(f"测试过程中发生异常: {e}")
sys.exit(1)
finally:
# 清理设备连接
if device:
logger.info("\n正在断开设备连接...")
try:
device.disconnect()
logger.info("✓ 设备已断开")
except Exception as e:
logger.warning(f"设备断开时出现警告: {e}")