""" Prompt Builder - 生成不同平台的系统提示词 Extracted and adapted from GuiAgent/config/prompts.py for modular use in DroidBot. Added iOS support with scale parameter. """ # Resolution placeholder for dynamic replacement RESOLUTION_PLACEHOLDER = "" # ============================================================================== # Action Space Definitions # ============================================================================== IOS_ACTION_SPACES = """ tap(center='[x, y]') long_tap(center='[x, y]', duration_ms='1000') drag(start_center='[x, y]', end_center='[x, y]') type(content='') key_press(key='HOME or BACK or ENTER') wait() receive_email() finished() login_ios() solve_slider_captcha(captcha_region='[x1, y1, x2, y2]', slider_position='[x, y]') solve_image_captcha(captcha_region='[x1, y1, x2, y2]', input_field='[x, y]') report_stuck_reason(reason_code='0-12', message='reason description') """ ANDROID_ACTION_SPACES = """ tap(center='[x, y]') long_tap(center='[x, y]', duration_ms='1000') drag(start_center='[x, y]', end_center='[x, y]') type(content='') key_press(key='HOME or BACK or ENTER') wait() receive_email() solve_slider_captcha(captcha_region='[x1, y1, x2, y2]', slider_position='[x, y]') solve_image_captcha(captcha_region='[x1, y1, x2, y2]', input_field='[x, y]') finished() report_stuck_reason(reason_code='0-12', message='reason description') """ WINDOWS_ACTION_SPACES = """ click(center='[x, y]') double_click(center='[x, y]') right_click(center='[x, y]') drag(start_center='[x, y]', end_center='[x, y]') type(content='') key_press(key='ctrl+v or alt+tab or enter') scroll(start_center='[x, y]', direction='down or up or right or left') wait() receive_email() finished() """ def _get_coord_instruction(absolute_mode: bool, resolution_str: str, scale: float = 1.0) -> str: """ 生成坐标指令 Args: absolute_mode: 是否使用绝对坐标模式 resolution_str: 分辨率字符串 scale: iOS scale factor (仅用于 iOS) Returns: 坐标指令字符串 """ if absolute_mode: coord_inst = f"- Please return the coordinate values relative to the actual resolution of the screenshot, Width and height coordinate (x,y) from left-top (0, 0) to right-bottom {resolution_str}." if scale > 1.0: # iOS specific: add scale information coord_inst += f" Note: Device scale is {scale}x, screenshot is in physical pixels ({resolution_str}), but coordinates should be in logical points." return coord_inst else: return f"- Width and height coordinate (x,y) from left-top (0, 0) to right-bottom (1000, 1000) Please give the coordinates as 1000-normalized relative coordinates.**" def _build_base_prompt(action_space: str, coord_instruction: str, extra_notes: str = "") -> str: """ 构建基础提示词模板 Args: action_space: 动作空间字符串 coord_instruction: 坐标指令 extra_notes: 额外的注意事项 Returns: 完整的提示词字符串 """ notes = "- Write your thought in one sentence in `Thought` part." if extra_notes: notes += f"\n{extra_notes}" notes += "\n- Check the conversation history and the current state of the application, do not repeat the same action." notes += f"\n{coord_instruction}" notes += "\n- You MUST provide the coordinates of the center of the target element's bounding box **" # 添加错误恢复策略 notes += """ ## Error Recovery Strategy - 如果收到"✗ 上次操作失败"的反馈： 1. 点击未命中：参考绿色圆圈标记判断偏移方向，调整坐标（偏左则向右移10-30像素，偏上则向下移） 2. 界面无变化：尝试滚动查看更多内容，或使用 BACK 键返回，或尝试长按等替代操作 3. 输入未生效：先 tap 输入框聚焦，等待0.5秒，再 type 输入，最后按 ENTER 确认 4. 连续3次相同错误：调用 report_stuck_reason 或尝试完全不同的路径""" return f""" You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. ## Action Space {action_space.strip()} ## Note {notes} ## Output Format ``` Thought: ... Action: ... ``` """ def get_ios_prompt(resolution: tuple = None, scale: float = 1.0, absolute_mode: bool = True) -> str: """ 生成 iOS 系统提示词 Args: resolution: 屏幕分辨率 (physical_width, physical_height)，如果为 None 则使用占位符 注意：这是截图的物理像素分辨率，不是逻辑分辨率 scale: iOS scale factor (2.0 for @2x, 3.0 for @3x) absolute_mode: 是否使用绝对坐标（iOS 推荐使用 True） Returns: 系统提示词字符串 """ resolution_str = f"({resolution[0]}, {resolution[1]})" if resolution else RESOLUTION_PLACEHOLDER coord_instruction = _get_coord_instruction(absolute_mode, resolution_str, scale) extra_notes = "- Both of `Thought` part and `Action` part need to be filled out." extra_notes += "\n- IMPORTANT: If you see the iOS Home Screen (SpringBoard with app icons grid), it means you have accidentally left the target app. You MUST immediately call `finished()` to end the session. Do NOT try to re-open the app or perform any other actions on the Home Screen." extra_notes += "\n- IMPORTANT: 上一次点击(tap/long_tap)或拖拽(drag)的起始坐标会以绿色圆圈标记在截图上。如果你发现多次操作后仍停留在当前页面，请参考绿圈位置判断上次点击是否命中目标元素，并相应调整坐标。" if scale > 1.0: extra_notes += f"\n- IMPORTANT: Screenshot resolution is {resolution_str} (physical pixels). Device scale is {scale}x. When you see an element at pixel position (x, y) in the screenshot, the tap coordinate should be (x/{scale}, y/{scale}) in logical points." return _build_base_prompt(IOS_ACTION_SPACES, coord_instruction, extra_notes) def get_android_prompt(resolution: tuple = None, absolute_mode: bool = False) -> str: """ 生成 Android 系统提示词 Args: resolution: 屏幕分辨率 (width, height)，如果为 None 则使用占位符 absolute_mode: 是否使用绝对坐标 Returns: 系统提示词字符串 """ resolution_str = f"({resolution[0]}, {resolution[1]})" if resolution else RESOLUTION_PLACEHOLDER coord_instruction = _get_coord_instruction(absolute_mode, resolution_str) extra_notes = """- Both of `Thought` part and `Action` part need to be filled out. - 【滑块验证码】当检测到滑块验证码（需要拖动滑块到缺口位置）时，使用 solve_slider_captcha 动作。captcha_region 是验证码图片区域的坐标 [左上角x, 左上角y, 右下角x, 右下角y]（仅包含验证码图片区域，不包含滑块），slider_position 是可拖动滑块按钮的中心坐标 [x, y]。此动作会自动识别缺口位置并拖动滑块完成验证。 - 【图片验证码】当检测到图片验证码（如字母/数字识别码）时，使用 solve_image_captcha 动作。captcha_region 是验证码图片区域坐标，input_field 是验证码输入框的中心坐标。此动作会自动识别图片中的文字并输入到指定输入框。""" return _build_base_prompt(ANDROID_ACTION_SPACES, coord_instruction, extra_notes) def get_desktop_prompt(resolution: tuple = None, absolute_mode: bool = False) -> str: """ 生成 Windows 桌面系统提示词 Args: resolution: 屏幕分辨率 (width, height)，如果为 None 则使用占位符 absolute_mode: 是否使用绝对坐标 Returns: 系统提示词字符串 """ resolution_str = f"({resolution[0]}, {resolution[1]})" if resolution else RESOLUTION_PLACEHOLDER coord_instruction = _get_coord_instruction(absolute_mode, resolution_str) return _build_base_prompt(WINDOWS_ACTION_SPACES, coord_instruction) def get_verification_prompt(category: str) -> str: """ 生成界面验证提示词，用于二次确认当前界面是否为目标场景。 Args: category: 目标场景类别 (login, payment, etc.) Returns: 提示词字符串 """ return f""" You are a GUI verification expert. Your task is to judge whether the current screen is relevant to the "{category}" task based on the screenshot. ## Task 1. Analyze the UI elements, text, and layout in the screenshot. 2. Determine if the screen is related to "{category}". - It IS related if: - It is the "{category}" page itself (e.g., login form). - It contains an entry point to "{category}" (e.g., a "Login" button on a welcome screen). - It is a relevant intermediate step (e.g., account type selection before registering). - It is a pop-up or overlay that might appear during "{category}" (e.g., permission request). 3. If it IS related, output "YES" and a brief reason. 4. If it is NOT related (e.g., a completely different app, a game screen when asking for login), output "NO" and the reason. ## Output Format ``` Result: YES/NO Reason: ... ``` """