fix: improve action parser compatibility

README_deploy.md

@@ -81,7 +81,9 @@ def add_box_token(input_string):
     return final_string
 
 client = OpenAI(
-    base_url="https:xxx",
+    # For OpenAI-compatible endpoints, include the /v1 suffix.
+    # Example: "https://your-endpoint.example.com/v1"
+    base_url="https://xxx/v1",
     api_key="hf_xxx"
 )
 

codes/README.md

@@ -60,12 +60,13 @@ print(pyautogui_code)
 
 ```python
 from PIL import Image, ImageDraw
+import ast
 import numpy as np
 import matplotlib.pyplot as plt
 
 image = Image.open("your_image_path.png")
 start_box = parsed_dict[0]["action_inputs"]["start_box"]
-coordinates = eval(start_box)
+coordinates = ast.literal_eval(start_box)
 x1 = int(coordinates[0] * original_image_width)
 y1 = int(coordinates[1] * original_image_height)
 draw = ImageDraw.Draw(image)
@@ -98,6 +99,20 @@ def parse_action_to_structure_output(
 **Description:**
 Parses output action instructions into structured dictionaries, automatically handling coordinate scaling and box/point format conversion.
 
+Supported coordinate/action forms include:
+
+```text
+Action: click(point='<point>200 300</point>')
+Action: click(start_box='(200,300)')
+Action: click(start_box='<|box_start|>(200,300)<|box_end|>')
+Action: scroll(start_box='(800,200)', end_box='(200,800)')
+<think_...>reasoning</think_...>click>point>point>200 300
+```
+
+For Qwen2.5-VL style models, coordinates are treated as absolute coordinates
+in the resized image and are normalized by `smart_resize`. For older
+relative-coordinate models, pass the appropriate `factor`.
+
 **Parameters:**
 - `text`: The output string
 - `factor`: Scaling factor
@@ -125,6 +140,11 @@ def parsing_response_to_pyautogui_code(
 **Description:**
 Converts structured actions into a pyautogui script string, supporting click, type, hotkey, drag, scroll, and more.
 
+`scroll` supports both mouse-wheel style actions with `direction` and
+gesture-style actions with `start_box` + `end_box`. Gesture-style scrolls are
+emitted as a `moveTo` + `dragTo` sequence, matching how mobile/emulator scrolls
+are usually executed.
+
 **Parameters:**
 - `responses`: Structured actions (dict or list of dicts)
 - `image_height`/`image_width`: Image height/width

codes/tests/action_parser_test.py

@@ -22,16 +22,83 @@ def test_parse_action(self):
     def test_parse_action_to_structure_output(self):
         text = "Thought: test\nAction: click(point='<point>200 300</point>')"
         actions = parse_action_to_structure_output(
-            text, factor=1000, origin_resized_height=224, origin_resized_width=224
+            text,
+            factor=1000,
+            origin_resized_height=224,
+            origin_resized_width=224,
+            min_pixels=1,
         )
         self.assertEqual(actions[0]['action_type'], 'click')
         self.assertIn('start_box', actions[0]['action_inputs'])
 
+    def test_parse_action_with_box_tokens(self):
+        text = (
+            "Thought: test\n"
+            "Action: click(start_box='<|box_start|>(112,224)<|box_end|>')"
+        )
+        actions = parse_action_to_structure_output(
+            text,
+            factor=1000,
+            origin_resized_height=224,
+            origin_resized_width=224,
+            min_pixels=1,
+        )
+        self.assertEqual(actions[0]["action_type"], "click")
+        self.assertEqual(
+            actions[0]["action_inputs"]["start_box"],
+            "[0.5, 1.0, 0.5, 1.0]",
+        )
+
+    def test_parse_doubao_seed_compact_action(self):
+        text = (
+            "<think_never_used_51bce0c785ca2f68081bfa7d91973934>"
+            "Click the orientation menu."
+            "</think_never_used_51bce0c785ca2f68081bfa7d91973934>"
+            "click>point>point>112 56"
+        )
+        actions = parse_action_to_structure_output(
+            text,
+            factor=1000,
+            origin_resized_height=224,
+            origin_resized_width=224,
+            min_pixels=1,
+        )
+        self.assertEqual(actions[0]["thought"], "Click the orientation menu.")
+        self.assertEqual(actions[0]["action_type"], "click")
+        self.assertEqual(
+            actions[0]["action_inputs"]["start_box"],
+            "[0.5, 0.25, 0.5, 0.25]",
+        )
+
     def test_parsing_response_to_pyautogui_code(self):
         responses = {"action_type": "hotkey", "action_inputs": {"hotkey": "ctrl v"}}
         code = parsing_response_to_pyautogui_code(responses, 224, 224)
         self.assertIn('pyautogui.hotkey', code)
 
+    def test_scroll_with_end_box_generates_drag_gesture(self):
+        responses = {
+            "action_type": "scroll",
+            "action_inputs": {
+                "start_box": "[0.8, 0.2, 0.8, 0.2]",
+                "end_box": "[0.2, 0.8, 0.2, 0.8]",
+            },
+        }
+        code = parsing_response_to_pyautogui_code(responses, 1000, 1000)
+        self.assertIn("pyautogui.moveTo(800.0, 200.0)", code)
+        self.assertIn("pyautogui.dragTo(200.0, 800.0, duration=1.0)", code)
+
+    def test_drag_accepts_two_point_tuples(self):
+        responses = {
+            "action_type": "drag",
+            "action_inputs": {
+                "start_box": ("897", "1208"),
+                "end_box": ("244", "1208"),
+            },
+        }
+        code = parsing_response_to_pyautogui_code(responses, 1, 1)
+        self.assertIn("pyautogui.moveTo(897.0, 1208.0)", code)
+        self.assertIn("pyautogui.dragTo(244.0, 1208.0, duration=1.0)", code)
+
 
 if __name__ == '__main__':
     unittest.main()

codes/ui_tars/action_parser.py

@@ -11,13 +11,12 @@
 
 
 def convert_point_to_coordinates(text, is_answer=False):
-    # 匹配 <bbox> 后面的四个数字
-    pattern = r"<point>(\d+)\s+(\d+)</point>"
+    # Match point tags emitted by the grounding prompts. Some models separate
+    # x/y with spaces, while others include a comma.
+    pattern = r"<point>\s*(-?\d+(?:\.\d+)?)\s*,?\s+(-?\d+(?:\.\d+)?)\s*</point>"
 
     def replace_match(match):
-        x1, y1 = map(int, match.groups())
-        x = (x1 + x1) // 2  # 使用截断取整
-        y = (y1 + y1) // 2  # 使用截断取整
+        x, y = (_format_coordinate(float(value)) for value in match.groups())
         if is_answer:
             return f"({x},{y})"  # 只返回 (x, y) 格式
         return f"({x},{y})"  # 返回带标签的格式
@@ -27,6 +26,67 @@ def replace_match(match):
     return re.sub(pattern, replace_match, text).strip()
 
 
+def _format_coordinate(value):
+    if value.is_integer():
+        return str(int(value))
+    return str(value)
+
+
+def _strip_coordinate_tokens(value):
+    return re.sub(r"</?\|?(?:box|point)_(?:start|end)\|?>", "", value)
+
+
+def _parse_coordinate_values(value):
+    if isinstance(value, (tuple, list)):
+        return [float(item) for item in value]
+
+    if value is None:
+        return []
+
+    value = _strip_coordinate_tokens(str(value).strip())
+    try:
+        literal = ast.literal_eval(value)
+        if isinstance(literal, (tuple, list)):
+            return [float(item) for item in literal]
+    except (SyntaxError, ValueError):
+        pass
+
+    return [
+        float(item)
+        for item in re.findall(r"-?\d+(?:\.\d+)?", value)
+    ]
+
+
+def _parse_coordinate_box(value):
+    values = _parse_coordinate_values(value)
+    if len(values) == 2:
+        return values[0], values[1], values[0], values[1]
+    if len(values) == 4:
+        return values[0], values[1], values[2], values[3]
+    raise ValueError(f"Expected 2 or 4 coordinate values, got {value!r}")
+
+
+def _normalize_seed_action_format(text):
+    """Convert doubao-seed compact actions into the normal Action: form."""
+    if "Action:" in text:
+        return text
+
+    match = re.search(
+        r"\b(click)\s*>\s*point\s*>\s*point\s*>\s*"
+        r"(-?\d+(?:\.\d+)?)\s*,?\s+(-?\d+(?:\.\d+)?)",
+        text,
+        re.IGNORECASE,
+    )
+    if not match:
+        return text
+
+    action, x, y = match.groups()
+    think_match = re.search(r"<think[^>]*>(.*?)</think[^>]*>", text, re.DOTALL)
+    thought = think_match.group(1).strip() if think_match else ""
+    prefix = f"Thought: {thought}\n" if thought else ""
+    return f"{prefix}Action: {action.lower()}(point='<point>{x} {y}</point>')"
+
+
 # 定义一个函数来解析每个 action
 def parse_action(action_str):
     try:
@@ -150,7 +210,7 @@ def parse_action_to_structure_output(text,
                                      model_type="qwen25vl",
                                      max_pixels=16384 * 28 * 28,
                                      min_pixels=100 * 28 * 28):
-    text = text.strip()
+    text = _normalize_seed_action_format(text.strip())
 
     if "<point>" in text:
         text = convert_point_to_coordinates(text)
@@ -240,15 +300,13 @@ def escape_quotes(match):
 
             if "start_box" in param_name or "end_box" in param_name:
                 ori_box = param
-                # Remove parentheses and split the string by commas
-                numbers = ori_box.replace("(", "").replace(")", "").split(",")
+                numbers = _parse_coordinate_values(ori_box)
 
                 # Convert to float and scale by 1000
                 # Qwen2.5vl output absolute coordinates, qwen2vl output relative coordinates
                 if model_type == "qwen25vl":
                     float_numbers = []
                     for num_idx, num in enumerate(numbers):
-                        num = float(num)
                         if (num_idx + 1) % 2 == 0:
                             float_numbers.append(
                                 float(num / smart_resize_height))
@@ -426,12 +484,10 @@ def parsing_response_to_pyautogui_code(responses,
             start_box = action_inputs.get("start_box")
             end_box = action_inputs.get("end_box")
             if start_box and end_box:
-                x1, y1, x2, y2 = eval(
-                    start_box)  # Assuming box is in [x1, y1, x2, y2]
+                x1, y1, x2, y2 = _parse_coordinate_box(start_box)
                 sx = round(float((x1 + x2) / 2) * image_width, 3)
                 sy = round(float((y1 + y2) / 2) * image_height, 3)
-                x1, y1, x2, y2 = eval(
-                    end_box)  # Assuming box is in [x1, y1, x2, y2]
+                x1, y1, x2, y2 = _parse_coordinate_box(end_box)
                 ex = round(float((x1 + x2) / 2) * image_width, 3)
                 ey = round(float((y1 + y2) / 2) * image_height, 3)
                 pyautogui_code += (
@@ -441,9 +497,9 @@ def parsing_response_to_pyautogui_code(responses,
         elif action_type == "scroll":
             # Parsing scroll action
             start_box = action_inputs.get("start_box")
+            end_box = action_inputs.get("end_box")
             if start_box:
-                x1, y1, x2, y2 = eval(
-                    start_box)  # Assuming box is in [x1, y1, x2, y2]
+                x1, y1, x2, y2 = _parse_coordinate_box(start_box)
                 x = round(float((x1 + x2) / 2) * image_width, 3)
                 y = round(float((y1 + y2) / 2) * image_height, 3)
 
@@ -454,7 +510,14 @@ def parsing_response_to_pyautogui_code(responses,
                 y = None
             direction = action_inputs.get("direction", "")
 
-            if x == None:
+            if start_box and end_box:
+                x1, y1, x2, y2 = _parse_coordinate_box(end_box)
+                ex = round(float((x1 + x2) / 2) * image_width, 3)
+                ey = round(float((y1 + y2) / 2) * image_height, 3)
+                pyautogui_code += (
+                    f"\npyautogui.moveTo({x}, {y})\n"
+                    f"\npyautogui.dragTo({ex}, {ey}, duration=1.0)\n")
+            elif x == None:
                 if "up" in direction.lower():
                     pyautogui_code += f"\npyautogui.scroll(5)"
                 elif "down" in direction.lower():
@@ -470,15 +533,8 @@ def parsing_response_to_pyautogui_code(responses,
         ]:
             # Parsing mouse click actions
             start_box = action_inputs.get("start_box")
-            start_box = str(start_box)
             if start_box:
-                start_box = eval(start_box)
-                if len(start_box) == 4:
-                    x1, y1, x2, y2 = start_box  # Assuming box is in [x1, y1, x2, y2]
-                elif len(start_box) == 2:
-                    x1, y1 = start_box
-                    x2 = x1
-                    y2 = y1
+                x1, y1, x2, y2 = _parse_coordinate_box(start_box)
                 x = round(float((x1 + x2) / 2) * image_width, 3)
                 y = round(float((y1 + y2) / 2) * image_height, 3)
                 if action_type == "left_single" or action_type == "click":