diff --git a/README_deploy.md b/README_deploy.md index 3b5f87d..e33f456 100644 --- a/README_deploy.md +++ b/README_deploy.md @@ -81,7 +81,9 @@ def add_box_token(input_string): return final_string client = OpenAI( - base_url="https:xxx", + # For OpenAI-compatible endpoints, include the /v1 suffix. + # Example: "https://your-endpoint.example.com/v1" + base_url="https://xxx/v1", api_key="hf_xxx" ) diff --git a/codes/README.md b/codes/README.md index 8da3b82..5b87ab1 100644 --- a/codes/README.md +++ b/codes/README.md @@ -60,12 +60,13 @@ print(pyautogui_code) ```python from PIL import Image, ImageDraw +import ast import numpy as np import matplotlib.pyplot as plt image = Image.open("your_image_path.png") start_box = parsed_dict[0]["action_inputs"]["start_box"] -coordinates = eval(start_box) +coordinates = ast.literal_eval(start_box) x1 = int(coordinates[0] * original_image_width) y1 = int(coordinates[1] * original_image_height) draw = ImageDraw.Draw(image) @@ -98,6 +99,20 @@ def parse_action_to_structure_output( **Description:** Parses output action instructions into structured dictionaries, automatically handling coordinate scaling and box/point format conversion. +Supported coordinate/action forms include: + +```text +Action: click(point='200 300') +Action: click(start_box='(200,300)') +Action: click(start_box='<|box_start|>(200,300)<|box_end|>') +Action: scroll(start_box='(800,200)', end_box='(200,800)') +reasoningclick>point>point>200 300 +``` + +For Qwen2.5-VL style models, coordinates are treated as absolute coordinates +in the resized image and are normalized by `smart_resize`. For older +relative-coordinate models, pass the appropriate `factor`. + **Parameters:** - `text`: The output string - `factor`: Scaling factor @@ -125,6 +140,11 @@ def parsing_response_to_pyautogui_code( **Description:** Converts structured actions into a pyautogui script string, supporting click, type, hotkey, drag, scroll, and more. +`scroll` supports both mouse-wheel style actions with `direction` and +gesture-style actions with `start_box` + `end_box`. Gesture-style scrolls are +emitted as a `moveTo` + `dragTo` sequence, matching how mobile/emulator scrolls +are usually executed. + **Parameters:** - `responses`: Structured actions (dict or list of dicts) - `image_height`/`image_width`: Image height/width diff --git a/codes/tests/action_parser_test.py b/codes/tests/action_parser_test.py index ca04cdc..92140da 100644 --- a/codes/tests/action_parser_test.py +++ b/codes/tests/action_parser_test.py @@ -22,16 +22,83 @@ def test_parse_action(self): def test_parse_action_to_structure_output(self): text = "Thought: test\nAction: click(point='200 300')" actions = parse_action_to_structure_output( - text, factor=1000, origin_resized_height=224, origin_resized_width=224 + text, + factor=1000, + origin_resized_height=224, + origin_resized_width=224, + min_pixels=1, ) self.assertEqual(actions[0]['action_type'], 'click') self.assertIn('start_box', actions[0]['action_inputs']) + def test_parse_action_with_box_tokens(self): + text = ( + "Thought: test\n" + "Action: click(start_box='<|box_start|>(112,224)<|box_end|>')" + ) + actions = parse_action_to_structure_output( + text, + factor=1000, + origin_resized_height=224, + origin_resized_width=224, + min_pixels=1, + ) + self.assertEqual(actions[0]["action_type"], "click") + self.assertEqual( + actions[0]["action_inputs"]["start_box"], + "[0.5, 1.0, 0.5, 1.0]", + ) + + def test_parse_doubao_seed_compact_action(self): + text = ( + "" + "Click the orientation menu." + "" + "click>point>point>112 56" + ) + actions = parse_action_to_structure_output( + text, + factor=1000, + origin_resized_height=224, + origin_resized_width=224, + min_pixels=1, + ) + self.assertEqual(actions[0]["thought"], "Click the orientation menu.") + self.assertEqual(actions[0]["action_type"], "click") + self.assertEqual( + actions[0]["action_inputs"]["start_box"], + "[0.5, 0.25, 0.5, 0.25]", + ) + def test_parsing_response_to_pyautogui_code(self): responses = {"action_type": "hotkey", "action_inputs": {"hotkey": "ctrl v"}} code = parsing_response_to_pyautogui_code(responses, 224, 224) self.assertIn('pyautogui.hotkey', code) + def test_scroll_with_end_box_generates_drag_gesture(self): + responses = { + "action_type": "scroll", + "action_inputs": { + "start_box": "[0.8, 0.2, 0.8, 0.2]", + "end_box": "[0.2, 0.8, 0.2, 0.8]", + }, + } + code = parsing_response_to_pyautogui_code(responses, 1000, 1000) + self.assertIn("pyautogui.moveTo(800.0, 200.0)", code) + self.assertIn("pyautogui.dragTo(200.0, 800.0, duration=1.0)", code) + + def test_drag_accepts_two_point_tuples(self): + responses = { + "action_type": "drag", + "action_inputs": { + "start_box": ("897", "1208"), + "end_box": ("244", "1208"), + }, + } + code = parsing_response_to_pyautogui_code(responses, 1, 1) + self.assertIn("pyautogui.moveTo(897.0, 1208.0)", code) + self.assertIn("pyautogui.dragTo(244.0, 1208.0, duration=1.0)", code) + if __name__ == '__main__': unittest.main() diff --git a/codes/ui_tars/action_parser.py b/codes/ui_tars/action_parser.py index 2b722f4..65972d3 100644 --- a/codes/ui_tars/action_parser.py +++ b/codes/ui_tars/action_parser.py @@ -11,13 +11,12 @@ def convert_point_to_coordinates(text, is_answer=False): - # 匹配 后面的四个数字 - pattern = r"(\d+)\s+(\d+)" + # Match point tags emitted by the grounding prompts. Some models separate + # x/y with spaces, while others include a comma. + pattern = r"\s*(-?\d+(?:\.\d+)?)\s*,?\s+(-?\d+(?:\.\d+)?)\s*" def replace_match(match): - x1, y1 = map(int, match.groups()) - x = (x1 + x1) // 2 # 使用截断取整 - y = (y1 + y1) // 2 # 使用截断取整 + x, y = (_format_coordinate(float(value)) for value in match.groups()) if is_answer: return f"({x},{y})" # 只返回 (x, y) 格式 return f"({x},{y})" # 返回带标签的格式 @@ -27,6 +26,67 @@ def replace_match(match): return re.sub(pattern, replace_match, text).strip() +def _format_coordinate(value): + if value.is_integer(): + return str(int(value)) + return str(value) + + +def _strip_coordinate_tokens(value): + return re.sub(r"", "", value) + + +def _parse_coordinate_values(value): + if isinstance(value, (tuple, list)): + return [float(item) for item in value] + + if value is None: + return [] + + value = _strip_coordinate_tokens(str(value).strip()) + try: + literal = ast.literal_eval(value) + if isinstance(literal, (tuple, list)): + return [float(item) for item in literal] + except (SyntaxError, ValueError): + pass + + return [ + float(item) + for item in re.findall(r"-?\d+(?:\.\d+)?", value) + ] + + +def _parse_coordinate_box(value): + values = _parse_coordinate_values(value) + if len(values) == 2: + return values[0], values[1], values[0], values[1] + if len(values) == 4: + return values[0], values[1], values[2], values[3] + raise ValueError(f"Expected 2 or 4 coordinate values, got {value!r}") + + +def _normalize_seed_action_format(text): + """Convert doubao-seed compact actions into the normal Action: form.""" + if "Action:" in text: + return text + + match = re.search( + r"\b(click)\s*>\s*point\s*>\s*point\s*>\s*" + r"(-?\d+(?:\.\d+)?)\s*,?\s+(-?\d+(?:\.\d+)?)", + text, + re.IGNORECASE, + ) + if not match: + return text + + action, x, y = match.groups() + think_match = re.search(r"]*>(.*?)]*>", text, re.DOTALL) + thought = think_match.group(1).strip() if think_match else "" + prefix = f"Thought: {thought}\n" if thought else "" + return f"{prefix}Action: {action.lower()}(point='{x} {y}')" + + # 定义一个函数来解析每个 action def parse_action(action_str): try: @@ -150,7 +210,7 @@ def parse_action_to_structure_output(text, model_type="qwen25vl", max_pixels=16384 * 28 * 28, min_pixels=100 * 28 * 28): - text = text.strip() + text = _normalize_seed_action_format(text.strip()) if "" in text: text = convert_point_to_coordinates(text) @@ -240,15 +300,13 @@ def escape_quotes(match): if "start_box" in param_name or "end_box" in param_name: ori_box = param - # Remove parentheses and split the string by commas - numbers = ori_box.replace("(", "").replace(")", "").split(",") + numbers = _parse_coordinate_values(ori_box) # Convert to float and scale by 1000 # Qwen2.5vl output absolute coordinates, qwen2vl output relative coordinates if model_type == "qwen25vl": float_numbers = [] for num_idx, num in enumerate(numbers): - num = float(num) if (num_idx + 1) % 2 == 0: float_numbers.append( float(num / smart_resize_height)) @@ -426,12 +484,10 @@ def parsing_response_to_pyautogui_code(responses, start_box = action_inputs.get("start_box") end_box = action_inputs.get("end_box") if start_box and end_box: - x1, y1, x2, y2 = eval( - start_box) # Assuming box is in [x1, y1, x2, y2] + x1, y1, x2, y2 = _parse_coordinate_box(start_box) sx = round(float((x1 + x2) / 2) * image_width, 3) sy = round(float((y1 + y2) / 2) * image_height, 3) - x1, y1, x2, y2 = eval( - end_box) # Assuming box is in [x1, y1, x2, y2] + x1, y1, x2, y2 = _parse_coordinate_box(end_box) ex = round(float((x1 + x2) / 2) * image_width, 3) ey = round(float((y1 + y2) / 2) * image_height, 3) pyautogui_code += ( @@ -441,9 +497,9 @@ def parsing_response_to_pyautogui_code(responses, elif action_type == "scroll": # Parsing scroll action start_box = action_inputs.get("start_box") + end_box = action_inputs.get("end_box") if start_box: - x1, y1, x2, y2 = eval( - start_box) # Assuming box is in [x1, y1, x2, y2] + x1, y1, x2, y2 = _parse_coordinate_box(start_box) x = round(float((x1 + x2) / 2) * image_width, 3) y = round(float((y1 + y2) / 2) * image_height, 3) @@ -454,7 +510,14 @@ def parsing_response_to_pyautogui_code(responses, y = None direction = action_inputs.get("direction", "") - if x == None: + if start_box and end_box: + x1, y1, x2, y2 = _parse_coordinate_box(end_box) + ex = round(float((x1 + x2) / 2) * image_width, 3) + ey = round(float((y1 + y2) / 2) * image_height, 3) + pyautogui_code += ( + f"\npyautogui.moveTo({x}, {y})\n" + f"\npyautogui.dragTo({ex}, {ey}, duration=1.0)\n") + elif x == None: if "up" in direction.lower(): pyautogui_code += f"\npyautogui.scroll(5)" elif "down" in direction.lower(): @@ -470,15 +533,8 @@ def parsing_response_to_pyautogui_code(responses, ]: # Parsing mouse click actions start_box = action_inputs.get("start_box") - start_box = str(start_box) if start_box: - start_box = eval(start_box) - if len(start_box) == 4: - x1, y1, x2, y2 = start_box # Assuming box is in [x1, y1, x2, y2] - elif len(start_box) == 2: - x1, y1 = start_box - x2 = x1 - y2 = y1 + x1, y1, x2, y2 = _parse_coordinate_box(start_box) x = round(float((x1 + x2) / 2) * image_width, 3) y = round(float((y1 + y2) / 2) * image_height, 3) if action_type == "left_single" or action_type == "click":