217 lines
9.1 KiB
Python
217 lines
9.1 KiB
Python
"""
|
||
Prompt Builder - 生成不同平台的系统提示词
|
||
|
||
Extracted and adapted from GuiAgent/config/prompts.py for modular use in DroidBot.
|
||
Added iOS support with scale parameter.
|
||
"""
|
||
|
||
# Resolution placeholder for dynamic replacement
|
||
RESOLUTION_PLACEHOLDER = "<RESOLUTION>"
|
||
|
||
# ==============================================================================
|
||
# Action Space Definitions
|
||
# ==============================================================================
|
||
|
||
IOS_ACTION_SPACES = """
|
||
tap(center='[x, y]')
|
||
long_tap(center='[x, y]', duration_ms='1000')
|
||
drag(start_center='[x, y]', end_center='[x, y]')
|
||
type(content='')
|
||
key_press(key='HOME or BACK or ENTER')
|
||
wait()
|
||
receive_email()
|
||
finished()
|
||
login_ios()
|
||
solve_slider_captcha(captcha_region='[x1, y1, x2, y2]', slider_position='[x, y]')
|
||
solve_image_captcha(captcha_region='[x1, y1, x2, y2]', input_field='[x, y]')
|
||
report_stuck_reason(reason_code='0-12', message='reason description')
|
||
"""
|
||
|
||
ANDROID_ACTION_SPACES = """
|
||
tap(center='[x, y]')
|
||
long_tap(center='[x, y]', duration_ms='1000')
|
||
drag(start_center='[x, y]', end_center='[x, y]')
|
||
type(content='')
|
||
key_press(key='HOME or BACK or ENTER')
|
||
wait()
|
||
receive_email()
|
||
solve_slider_captcha(captcha_region='[x1, y1, x2, y2]', slider_position='[x, y]')
|
||
solve_image_captcha(captcha_region='[x1, y1, x2, y2]', input_field='[x, y]')
|
||
finished()
|
||
report_stuck_reason(reason_code='0-12', message='reason description')
|
||
"""
|
||
|
||
WINDOWS_ACTION_SPACES = """
|
||
click(center='[x, y]')
|
||
double_click(center='[x, y]')
|
||
right_click(center='[x, y]')
|
||
drag(start_center='[x, y]', end_center='[x, y]')
|
||
type(content='')
|
||
key_press(key='ctrl+v or alt+tab or enter')
|
||
scroll(start_center='[x, y]', direction='down or up or right or left')
|
||
wait()
|
||
receive_email()
|
||
finished()
|
||
"""
|
||
|
||
|
||
def _get_coord_instruction(absolute_mode: bool, resolution_str: str, scale: float = 1.0) -> str:
|
||
"""
|
||
生成坐标指令
|
||
|
||
Args:
|
||
absolute_mode: 是否使用绝对坐标模式
|
||
resolution_str: 分辨率字符串
|
||
scale: iOS scale factor (仅用于 iOS)
|
||
|
||
Returns:
|
||
坐标指令字符串
|
||
"""
|
||
if absolute_mode:
|
||
coord_inst = f"- Please return the coordinate values relative to the actual resolution of the screenshot, Width and height coordinate (x,y) from left-top (0, 0) to right-bottom {resolution_str}."
|
||
if scale > 1.0:
|
||
# iOS specific: add scale information
|
||
coord_inst += f" Note: Device scale is {scale}x, screenshot is in physical pixels ({resolution_str}), but coordinates should be in logical points."
|
||
return coord_inst
|
||
else:
|
||
return f"- Width and height coordinate (x,y) from left-top (0, 0) to right-bottom (1000, 1000) Please give the coordinates as 1000-normalized relative coordinates.**"
|
||
|
||
|
||
def _build_base_prompt(action_space: str, coord_instruction: str, extra_notes: str = "") -> str:
|
||
"""
|
||
构建基础提示词模板
|
||
|
||
Args:
|
||
action_space: 动作空间字符串
|
||
coord_instruction: 坐标指令
|
||
extra_notes: 额外的注意事项
|
||
|
||
Returns:
|
||
完整的提示词字符串
|
||
"""
|
||
notes = "- Write your thought in one sentence in `Thought` part."
|
||
if extra_notes:
|
||
notes += f"\n{extra_notes}"
|
||
notes += "\n- Check the conversation history and the current state of the application, do not repeat the same action."
|
||
notes += f"\n{coord_instruction}"
|
||
notes += "\n- You MUST provide the coordinates of the center of the target element's bounding box **"
|
||
|
||
# 添加错误恢复策略
|
||
notes += """
|
||
|
||
## Error Recovery Strategy
|
||
- 如果收到"✗ 上次操作失败"的反馈：
|
||
1. 点击未命中：参考绿色圆圈标记判断偏移方向，调整坐标（偏左则向右移10-30像素，偏上则向下移）
|
||
2. 界面无变化：尝试滚动查看更多内容，或使用 BACK 键返回，或尝试长按等替代操作
|
||
3. 输入未生效：先 tap 输入框聚焦，等待0.5秒，再 type 输入，最后按 ENTER 确认
|
||
4. 连续3次相同错误：调用 report_stuck_reason 或尝试完全不同的路径"""
|
||
|
||
return f"""
|
||
You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
|
||
## Action Space
|
||
{action_space.strip()}
|
||
## Note
|
||
{notes}
|
||
## Output Format
|
||
```
|
||
Thought: ...
|
||
Action: ...
|
||
```
|
||
"""
|
||
|
||
|
||
def get_ios_prompt(resolution: tuple = None, scale: float = 1.0, absolute_mode: bool = True) -> str:
|
||
"""
|
||
生成 iOS 系统提示词
|
||
|
||
Args:
|
||
resolution: 屏幕分辨率 (physical_width, physical_height)，如果为 None 则使用占位符
|
||
注意：这是截图的物理像素分辨率，不是逻辑分辨率
|
||
scale: iOS scale factor (2.0 for @2x, 3.0 for @3x)
|
||
absolute_mode: 是否使用绝对坐标（iOS 推荐使用 True）
|
||
|
||
Returns:
|
||
系统提示词字符串
|
||
"""
|
||
resolution_str = f"({resolution[0]}, {resolution[1]})" if resolution else RESOLUTION_PLACEHOLDER
|
||
|
||
coord_instruction = _get_coord_instruction(absolute_mode, resolution_str, scale)
|
||
|
||
extra_notes = "- Both of `Thought` part and `Action` part need to be filled out."
|
||
extra_notes += "\n- IMPORTANT: If you see the iOS Home Screen (SpringBoard with app icons grid), it means you have accidentally left the target app. You MUST immediately call `finished()` to end the session. Do NOT try to re-open the app or perform any other actions on the Home Screen."
|
||
extra_notes += "\n- IMPORTANT: 上一次点击(tap/long_tap)或拖拽(drag)的起始坐标会以绿色圆圈标记在截图上。如果你发现多次操作后仍停留在当前页面，请参考绿圈位置判断上次点击是否命中目标元素，并相应调整坐标。"
|
||
if scale > 1.0:
|
||
extra_notes += f"\n- IMPORTANT: Screenshot resolution is {resolution_str} (physical pixels). Device scale is {scale}x. When you see an element at pixel position (x, y) in the screenshot, the tap coordinate should be (x/{scale}, y/{scale}) in logical points."
|
||
|
||
return _build_base_prompt(IOS_ACTION_SPACES, coord_instruction, extra_notes)
|
||
|
||
|
||
def get_android_prompt(resolution: tuple = None, absolute_mode: bool = False) -> str:
|
||
"""
|
||
生成 Android 系统提示词
|
||
|
||
Args:
|
||
resolution: 屏幕分辨率 (width, height)，如果为 None 则使用占位符
|
||
absolute_mode: 是否使用绝对坐标
|
||
|
||
Returns:
|
||
系统提示词字符串
|
||
"""
|
||
resolution_str = f"({resolution[0]}, {resolution[1]})" if resolution else RESOLUTION_PLACEHOLDER
|
||
|
||
coord_instruction = _get_coord_instruction(absolute_mode, resolution_str)
|
||
|
||
extra_notes = """- Both of `Thought` part and `Action` part need to be filled out.
|
||
- 【滑块验证码】当检测到滑块验证码（需要拖动滑块到缺口位置）时，使用 solve_slider_captcha 动作。captcha_region 是验证码图片区域的坐标 [左上角x, 左上角y, 右下角x, 右下角y]（仅包含验证码图片区域，不包含滑块），slider_position 是可拖动滑块按钮的中心坐标 [x, y]。此动作会自动识别缺口位置并拖动滑块完成验证。
|
||
- 【图片验证码】当检测到图片验证码（如字母/数字识别码）时，使用 solve_image_captcha 动作。captcha_region 是验证码图片区域坐标，input_field 是验证码输入框的中心坐标。此动作会自动识别图片中的文字并输入到指定输入框。"""
|
||
return _build_base_prompt(ANDROID_ACTION_SPACES, coord_instruction, extra_notes)
|
||
|
||
|
||
def get_desktop_prompt(resolution: tuple = None, absolute_mode: bool = False) -> str:
|
||
"""
|
||
生成 Windows 桌面系统提示词
|
||
|
||
Args:
|
||
resolution: 屏幕分辨率 (width, height)，如果为 None 则使用占位符
|
||
absolute_mode: 是否使用绝对坐标
|
||
|
||
Returns:
|
||
系统提示词字符串
|
||
"""
|
||
resolution_str = f"({resolution[0]}, {resolution[1]})" if resolution else RESOLUTION_PLACEHOLDER
|
||
coord_instruction = _get_coord_instruction(absolute_mode, resolution_str)
|
||
|
||
return _build_base_prompt(WINDOWS_ACTION_SPACES, coord_instruction)
|
||
|
||
|
||
def get_verification_prompt(category: str) -> str:
|
||
"""
|
||
生成界面验证提示词，用于二次确认当前界面是否为目标场景。
|
||
|
||
Args:
|
||
category: 目标场景类别 (login, payment, etc.)
|
||
|
||
Returns:
|
||
提示词字符串
|
||
"""
|
||
return f"""
|
||
You are a GUI verification expert. Your task is to judge whether the current screen is relevant to the "{category}" task based on the screenshot.
|
||
|
||
## Task
|
||
1. Analyze the UI elements, text, and layout in the screenshot.
|
||
2. Determine if the screen is related to "{category}".
|
||
- It IS related if:
|
||
- It is the "{category}" page itself (e.g., login form).
|
||
- It contains an entry point to "{category}" (e.g., a "Login" button on a welcome screen).
|
||
- It is a relevant intermediate step (e.g., account type selection before registering).
|
||
- It is a pop-up or overlay that might appear during "{category}" (e.g., permission request).
|
||
3. If it IS related, output "YES" and a brief reason.
|
||
4. If it is NOT related (e.g., a completely different app, a game screen when asking for login), output "NO" and the reason.
|
||
|
||
## Output Format
|
||
```
|
||
Result: YES/NO
|
||
Reason: ...
|
||
```
|
||
"""
|