|
| 1 | +"""A yes/no oracle for blocks on the table. |
| 2 | +
|
| 3 | +Arrange some blocks, then type a yes/no question. The model drives the arm to |
| 4 | +inspect the scene with the wrist camera, decides, and the arm nods (yes) or |
| 5 | +shakes its head (no) with fixed gestures. The arm rehomes and disables between |
| 6 | +questions, so start it resting on the table. |
| 7 | +
|
| 8 | + uv run --project examples/python examples/applications/block-oracle/block_oracle.py --model gpt-6-astra |
| 9 | +""" |
| 10 | + |
| 11 | +import argparse |
| 12 | +import asyncio |
| 13 | +import base64 |
| 14 | +import json |
| 15 | +import re |
| 16 | +import time |
| 17 | +from collections.abc import Awaitable, Mapping |
| 18 | +from pathlib import Path |
| 19 | +from typing import Any |
| 20 | + |
| 21 | +from hebbian_robot_use_client.contract.station_api import ArmMode |
| 22 | +from hebbian_robot_use_client.environment import load_environment |
| 23 | +from hebbian_robot_use_client.station_client import Lease |
| 24 | +from hebbian_robot_use_client.station_types import CommandResult |
| 25 | +from openai import AsyncOpenAI |
| 26 | + |
| 27 | +from hebbian_rua import RobotTools, StationClient |
| 28 | +from hebbian_rua.openai import OpenAIRobotTools |
| 29 | + |
| 30 | +ARM = "follower" |
| 31 | +CAMERA = "table" |
| 32 | +MAX_TURNS = 20 |
| 33 | +GESTURE_SPEED = 0.8 # rad/s; the profile caps all motion anyway |
| 34 | +# Gripper 22 cm forward, 14 cm up, tilted ~30 degrees below horizontal, facing the |
| 35 | +# user. Chosen so the nod keeps 0.77 rad from the wrist limit on this arm. |
| 36 | +PRESENT_JOINTS = [0.0, -1.341, 0.926, 0.955, 0.049] |
| 37 | +NOD_RADIANS = 0.3 # wrist_flex swing |
| 38 | +SHAKE_RADIANS = 0.35 # shoulder_pan swing |
| 39 | +RUNS = Path(__file__).with_name("runs") # frames and moves per question |
| 40 | +ANSWER = re.compile(r"ANSWER:\s*(YES|NO)\b", re.IGNORECASE) |
| 41 | + |
| 42 | +DEFAULT_WORKSPACE = "the bluish-green Seeed Studio mat directly in front of the arm" |
| 43 | + |
| 44 | +INSTRUCTIONS = """You are a yes/no oracle. The user has arranged blocks on the |
| 45 | +workspace and asked a question about them. The workspace is {workspace}. Every |
| 46 | +question is only about objects resting on the workspace: ignore everything else the |
| 47 | +camera sees, such as the desk beyond the workspace, the floor, furniture, devices, |
| 48 | +cables and the robot itself. An object counts only if it rests on the workspace. |
| 49 | +
|
| 50 | +Inspect before answering: move the arm so the wrist camera covers the whole |
| 51 | +workspace, for example lift to about 15-20 cm above it and look from the left, centre |
| 52 | +and right of the reachable area. Find the workspace edges in the images and look |
| 53 | +closer wherever you are unsure. From above, blocks stacked on each other look like |
| 54 | +a single block, so also look at every block from a low, side-on angle (camera about |
| 55 | +5-8 cm above the table, pointing roughly horizontally) to judge heights and stacking |
| 56 | +before counting. Rules: pass max_joint_speed of 0.6 or less on every |
| 57 | +move, keep the gripper at least 5 cm above the table, never touch or move the blocks, |
| 58 | +and do not use the gripper. |
| 59 | +
|
| 60 | +When you are confident, stop calling tools. Reply with one sentence stating how many |
| 61 | +blocks are on the workspace, where, and which are stacked, then a final line that is exactly |
| 62 | +'ANSWER: YES' or 'ANSWER: NO'. Robot context: """ |
| 63 | + |
| 64 | + |
| 65 | +async def inspect_and_decide( |
| 66 | + client: AsyncOpenAI, robot: RobotTools, model: str, workspace: str, question: str |
| 67 | +) -> tuple[str | None, str]: |
| 68 | + """Let the model drive the arm until it answers; return (YES/NO or None, text).""" |
| 69 | + robot_tools = OpenAIRobotTools(robot) |
| 70 | + instructions = INSTRUCTIONS.format(workspace=workspace) + json.dumps( |
| 71 | + await robot.context() |
| 72 | + ) |
| 73 | + messages: list = [{"role": "user", "content": question}] |
| 74 | + run = RUNS / time.strftime("%Y%m%d-%H%M%S") |
| 75 | + run.mkdir(parents=True) |
| 76 | + (run / "question.txt").write_text(question + "\n") |
| 77 | + for number in range(MAX_TURNS): |
| 78 | + observation = await robot_tools.observe() |
| 79 | + messages.append(observation) |
| 80 | + save_frames(observation, run / f"turn{number:02d}") |
| 81 | + response = await client.responses.create( |
| 82 | + model=model, |
| 83 | + instructions=instructions, |
| 84 | + tools=robot_tools.tools, |
| 85 | + input=messages, |
| 86 | + ) |
| 87 | + with (run / "log.txt").open("a") as log: |
| 88 | + for output in response.output: |
| 89 | + if output.type == "function_call": |
| 90 | + log.write(f"turn{number:02d} {output.name} {output.arguments}\n") |
| 91 | + log.write(f"turn{number:02d} text: {response.output_text}\n") |
| 92 | + turn = await robot_tools.execute(response) |
| 93 | + messages.extend(turn.messages) |
| 94 | + if turn.should_stop: |
| 95 | + return None, "Robot execution stopped after a fault or uncertain outcome." |
| 96 | + if not turn.results: |
| 97 | + match = ANSWER.search(response.output_text) |
| 98 | + return (match.group(1).upper() if match else None), response.output_text |
| 99 | + return None, f"No answer after {MAX_TURNS} turns." |
| 100 | + |
| 101 | + |
| 102 | +def save_frames(observation: Mapping[str, Any], prefix: Path) -> None: |
| 103 | + """Write the camera images the model is about to see, for replaying a run.""" |
| 104 | + images = [ |
| 105 | + item["image_url"] |
| 106 | + for item in observation["content"] |
| 107 | + if item["type"] == "input_image" |
| 108 | + ] |
| 109 | + for index, url in enumerate(images): |
| 110 | + jpeg = base64.b64decode(url.split(",", 1)[1]) |
| 111 | + prefix.with_name(f"{prefix.name}-{index}.jpg").write_bytes(jpeg) |
| 112 | + |
| 113 | + |
| 114 | +async def gesture(lease: Lease, answer: str) -> None: |
| 115 | + """Move to the presenting pose, then nod (YES) or shake (NO) three times.""" |
| 116 | + moving = lease.arm(ARM) |
| 117 | + joint, swing = (3, NOD_RADIANS) if answer == "YES" else (0, SHAKE_RADIANS) |
| 118 | + poses = [PRESENT_JOINTS] |
| 119 | + for _ in range(3): |
| 120 | + for offset in (swing, -swing): |
| 121 | + pose = list(PRESENT_JOINTS) |
| 122 | + pose[joint] += offset |
| 123 | + poses.append(pose) |
| 124 | + for pose in [*poses, PRESENT_JOINTS]: |
| 125 | + await expect(moving.move_to_joints(pose, max_joint_speed=GESTURE_SPEED)) |
| 126 | + |
| 127 | + |
| 128 | +async def expect(command: Awaitable[CommandResult]) -> None: |
| 129 | + result = await command |
| 130 | + if not result.completed: |
| 131 | + raise SystemExit(f"Move did not complete: {result.outcome} {result.detail}") |
| 132 | + |
| 133 | + |
| 134 | +async def ask( |
| 135 | + station: StationClient, |
| 136 | + client: AsyncOpenAI, |
| 137 | + model: str, |
| 138 | + workspace: str, |
| 139 | + question: str, |
| 140 | +) -> None: |
| 141 | + arm = station.arm(ARM) |
| 142 | + async with station.holding_lease(question, required_cameras=[CAMERA]) as lease: |
| 143 | + if (await arm.state()).mode != ArmMode.holding: |
| 144 | + await expect(arm.enable()) # the rest pose becomes the rehome target |
| 145 | + try: |
| 146 | + robot = RobotTools(station, lease, ARM, [CAMERA]) |
| 147 | + answer, text = await inspect_and_decide( |
| 148 | + client, robot, model, workspace, question |
| 149 | + ) |
| 150 | + print(f"\n{text.strip()}\n") |
| 151 | + if answer: |
| 152 | + await gesture(lease, answer) |
| 153 | + finally: |
| 154 | + await expect(lease.arm(ARM).rehome()) |
| 155 | + await expect(arm.disable()) # the rest pose is supported by the table |
| 156 | + |
| 157 | + |
| 158 | +async def main() -> None: |
| 159 | + parser = argparse.ArgumentParser(description=__doc__) |
| 160 | + parser.add_argument("--model", required=True, help="OpenAI model ID") |
| 161 | + parser.add_argument( |
| 162 | + "--workspace", |
| 163 | + default=DEFAULT_WORKSPACE, |
| 164 | + help="Where questions apply; objects elsewhere are ignored", |
| 165 | + ) |
| 166 | + arguments = parser.parse_args() |
| 167 | + load_environment() |
| 168 | + async with StationClient() as station, AsyncOpenAI() as client: |
| 169 | + while True: |
| 170 | + question = input("Ask a yes/no question about the blocks (blank to quit): ") |
| 171 | + if not question.strip(): |
| 172 | + break |
| 173 | + await ask(station, client, arguments.model, arguments.workspace, question) |
| 174 | + |
| 175 | + |
| 176 | +if __name__ == "__main__": |
| 177 | + asyncio.run(main()) |
0 commit comments