{"name":"Embodied First Exam","short":"EFE","tagline":"Code agents have passed many exams in the digital world. What is their first exam before entering the physical world?","summary":"The first exam for code agents in the physical world. Each question is a frozen simulation scene with a goal written for a person; the answer is a program, marked by replaying it in a fresh simulator. The questions come from many benchmark suites and are sat in two modes: privileged, with the simulator open to the agent, and standard, with only the robot.","build":{"public":true,"at":"2026-11-21T04:04:24+00:00"},"task_set":{"commit":"88dc065f8313972eebc77ea83a313b8388eb7858","date":"2026-11-20T15:26:38-07:00"},"task_alias":{"embodied-olympics-fun/basketball_dribble_realgrasp":"embodied-olympics-fun/basketball_dribble","embodied-olympics-fun/bowling_realgrasp":"embodied-olympics-fun/bowling","embodied-olympics-fun/catch_and_shoot_realgrasp":"embodied-olympics-fun/catch_and_shoot","embodied-olympics-fun/catch_realgrasp":"embodied-olympics-fun/catch","embodied-olympics-fun/discus_realgrasp":"embodied-olympics-fun/discus","embodied-olympics-fun/dunk_realgrasp":"embodied-olympics-fun/dunk","embodied-olympics-fun/free_throw_realgrasp":"embodied-olympics-fun/free_throw","embodied-olympics-fun/golf_putt_realgrasp":"embodied-olympics-fun/golf_putt","embodied-olympics-fun/javelin_realgrasp":"embodied-olympics-fun/javelin","embodied-olympics-fun/juggle_three_realgrasp":"embodied-olympics-fun/juggle_three","embodied-olympics-fun/juggling_feet_realgrasp":"embodied-olympics-fun/juggling_feet","embodied-olympics-fun/juggling_realgrasp":"embodied-olympics-fun/juggling","embodied-olympics-fun/layup_realgrasp":"embodied-olympics-fun/layup","embodied-olympics-fun/paddle_bounce_realgrasp":"embodied-olympics-fun/paddle_bounce","embodied-olympics-fun/pitch_target_realgrasp":"embodied-olympics-fun/pitch_target","embodied-olympics-fun/shot_put_realgrasp":"embodied-olympics-fun/shot_put","embodied-olympics-fun/table_tennis_return_realgrasp":"embodied-olympics-fun/table_tennis_return","embodied-olympics-fun/tennis_return_realgrasp":"embodied-olympics-fun/tennis_return","embodied-olympics-fun/tennis_serve_realgrasp":"embodied-olympics-fun/tennis_serve","embodied-olympics-fun/toes_to_bar_realgrasp":"embodied-olympics-fun/toes_to_bar","embodied-olympics-fun/volleyball_serve_realgrasp":"embodied-olympics-fun/volleyball_serve","embodied-olympics-fun/wall_ball_realgrasp":"embodied-olympics-fun/wall_ball","embodied-olympics-match/badminton_match_i01":"embodied-olympics-match/badminton_match_i04","embodied-olympics-match/badminton_match_i02":"embodied-olympics-match/badminton_match_i04","embodied-olympics-match/fencing_bout_i01":"embodied-olympics-match/fencing_bout_i02","embodied-olympics-match/soccerssl_match":"embodied-olympics-match/soccerssl_match_i01","embodied-olympics-match/tabletennis_match_i01":"embodied-olympics-match/tabletennis_match_i04","embodied-olympics-match/tabletennis_match_i02":"embodied-olympics-match/tabletennis_match_i04","embodied-olympics-match/tabletennis_match_i03":"embodied-olympics-match/tabletennis_match_i04","ikea-mujoco/eket":"roboarc/ikea_eket","ikea-mujoco/gladom":"roboarc/ikea_gladom","ikea-mujoco/ikea365":"roboarc/ikea_ikea365","ikea-mujoco/lack":"roboarc/ikea_lack","ikea-mujoco/stegesund":"roboarc/ikea_stegesund","ikea-mujoco/svallet":"roboarc/ikea_svallet","ikea-mujoco/variera":"roboarc/ikea_variera","ikea-mujoco/vesken":"roboarc/ikea_vesken","roboarc/eket":"roboarc/ikea_eket","roboarc/gladom":"roboarc/ikea_gladom","roboarc/ikea365":"roboarc/ikea_ikea365","roboarc/lack":"roboarc/ikea_lack","roboarc/stegesund":"roboarc/ikea_stegesund","roboarc/svallet":"roboarc/ikea_svallet","roboarc/variera":"roboarc/ikea_variera","roboarc/vesken":"roboarc/ikea_vesken"},"totals":{"suites":35,"instances":1156,"task_dirs":2294,"core_dirs":951,"core_tasks":481,"core_modes":{"privileged":416,"standard":481,"realtime":54},"default_run":481,"folders":35,"robots":47,"simulators":23,"graded_trials":10461},"suites":[{"id":"behavior-1k","name":"BEHAVIOR-1K","code":"B1K","origin":"adapted","example":false,"blurb":"Full household activities in house-scale scenes, for a mobile two-armed robot.","description":"100 full-length household activities in house-scale scenes. Agents must combine high-level reasoning, long-horizon navigation, and dexterous bimanual manipulation. Scored by average task success with BDDL partial credit.","edition":"2026 BEHAVIOR Challenge","simulator":"OmniGibson (NVIDIA Isaac Sim)","links":{"challenge":"https://behavior.stanford.edu/challenge/index.html","task_gallery":"https://behavior.stanford.edu/challenge/tasks/index.html","dataset_docs":"https://behavior.stanford.edu/challenge/dataset.html","call_for_participation":"https://behavior.stanford.edu/challenge/call_for_participation.html","leaderboard":"https://huggingface.co/spaces/behavior-1k/2026-challenge-leaderboard","submission":"https://behavior-1k-2026-challenge-leaderboard.hf.space/submit","repository":"https://github.com/StanfordVL/BEHAVIOR-1K","paper":"https://arxiv.org/abs/2403.09227","project_page":"https://behavior.stanford.edu/"},"robots":[{"id":"r1pro","name":"Galaxea R1 Pro","body":"mobile"}],"body":"mobile","bodies":{"mobile":100},"instances":100,"task_dirs":200,"modes":{"privileged":100,"standard":100},"oracle":{"full":0,"partial":2,"none":98},"protocols":["eai-standard/2.2"],"budgets_min":[120],"tags":{"perception-understanding":64,"planning-reasoning":97,"control-coordination":100,"feedback-adaptation":54,"mobile-whole-body-manipulation":100,"navigation-exploration":37},"on_site":true,"preview":[{"task":"setting_the_table","title":"Setting The Table","cover":true,"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/setting_the_table.jpg?v=71c83842","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/setting_the_table.mp4?v=f58c84c1","motion":true,"aspect":1.778,"seconds":30},{"task":"cook_brussels_sprouts","title":"Cook Brussels Sprouts","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/cook_brussels_sprouts.jpg?v=e8c647d8","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/cook_brussels_sprouts.mp4?v=252f736c","motion":true,"aspect":1.778,"seconds":30},{"task":"make_pizza","title":"Make Pizza","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/make_pizza.jpg?v=8ca854e7","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/make_pizza.mp4?v=eb74a8a5","motion":true,"aspect":1.778,"seconds":30},{"task":"slicing_vegetables","title":"Slicing Vegetables","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/slicing_vegetables.jpg?v=59b5c559","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/slicing_vegetables.mp4?v=a97a3b21","motion":true,"aspect":1.778,"seconds":30},{"task":"canning_food","title":"Canning Food","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/canning_food.jpg?v=d06fa98e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/canning_food.mp4?v=adbc94c2","motion":true,"aspect":1.778,"seconds":30},{"task":"setup_a_bar_for_a_cocktail_party","title":"Setup A Bar For A Cocktail Party","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/setup_a_bar_for_a_cocktail_party.jpg?v=50bbae22","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/setup_a_bar_for_a_cocktail_party.mp4?v=55d34517","motion":true,"aspect":1.778,"seconds":30},{"task":"putting_up_Christmas_decorations_inside","title":"Putting Up Christmas Decorations Inside","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/putting_up_Christmas_decorations_inside.jpg?v=d713afec","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/putting_up_Christmas_decorations_inside.mp4?v=8a6999bc","motion":true,"aspect":1.778,"seconds":30},{"task":"assembling_gift_baskets","title":"Assembling Gift Baskets","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/assembling_gift_baskets.jpg?v=d99deb25","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/assembling_gift_baskets.mp4?v=5a4baebd","motion":true,"aspect":1.778,"seconds":30}],"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":71,"success":2,"trials":71},"standard":{"done":71,"success":0,"trials":71}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":71,"success":6,"trials":71},"standard":{"done":71,"success":4,"trials":71}},"claude_code-fable_5_2-high":{"privileged":{"done":71,"success":36,"trials":71},"standard":{"done":71,"success":26,"trials":71}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":71,"success":1,"trials":71},"standard":{"done":71,"success":0,"trials":71}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":71,"success":13,"trials":71},"standard":{"done":71,"success":0,"trials":71}},"claude_code-opus_5_5-high":{"privileged":{"done":71,"success":22,"trials":71},"standard":{"done":71,"success":5,"trials":71}},"codex-gpt6_1_sol-high":{"privileged":{"done":71,"success":38,"trials":71},"standard":{"done":71,"success":13,"trials":71}},"codex-gpt6_2_astra-high":{"privileged":{"done":71,"success":67,"trials":71},"standard":{"done":71,"success":40,"trials":71}},"codex-gpt6_astra-high":{"privileged":{"done":71,"success":36,"trials":71},"standard":{"done":71,"success":14,"trials":71}},"codex-gpt6_luna-xhigh":{"privileged":{"done":71,"success":0,"trials":71},"standard":{"done":71,"success":0,"trials":71}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":71,"success":3,"trials":71},"standard":{"done":71,"success":0,"trials":71}}},"solved":{"privileged":96,"standard":51},"attempted":{"privileged":100,"standard":100},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":1464.19,"est_n":142,"agent_s":1323485,"agent_n":142},"antigravity_cli-gemini_4_argon-high":{"est_usd":956.8,"est_n":142,"agent_s":869673,"agent_n":142},"claude_code-fable_5_2-high":{"est_usd":2124.43,"est_n":141,"agent_s":704785,"agent_n":139},"claude_code-glm_5_3_flash-high":{"est_usd":111.42,"est_n":142,"agent_s":1585657,"agent_n":142},"claude_code-kimi_3_1_pro-high":{"est_usd":1345.94,"est_n":142,"agent_s":909032,"agent_n":142},"claude_code-opus_5_5-high":{"est_usd":1319.12,"est_n":142,"agent_s":761305,"agent_n":142},"codex-gpt6_1_sol-high":{"est_usd":917.33,"est_n":142,"agent_s":832113,"agent_n":142},"codex-gpt6_2_astra-high":{"est_usd":1549.53,"est_n":142,"agent_s":622141,"agent_n":142},"codex-gpt6_astra-high":{"est_usd":6707.66,"est_n":141,"agent_s":1079730,"agent_n":141},"codex-gpt6_luna-xhigh":{"est_usd":143.0,"est_n":142,"agent_s":1532008,"agent_n":142},"dsh-deepseek_v4_1_flash-high":{"est_usd":53.54,"est_n":142,"agent_s":757521,"agent_n":142}}},{"id":"robolab","name":"RoboLab","code":"RLB","origin":"adapted","example":false,"blurb":"Tabletop tasks that test language grounding, from colours and counts to relations and order.","description":"53 of its 120 tabletop manipulation tasks for a fixed-base arm, built to test language grounding and goal specification rather than locomotion: colour and size references, counting, spatial relations, sorting, ordered stacking, reorientation. Success is a composable predicate over object state, with partial credit from scored subtasks.","edition":"v0.3.1 (2026-08-11)","simulator":"Isaac Lab 2.3.2 / Isaac Sim 5.1 (NVIDIA)","links":{"paper":"https://arxiv.org/abs/2604.09860","project_page":"https://research.nvidia.com/labs/srl/projects/robolab","leaderboard":"https://research.nvidia.com/labs/srl/projects/robolab/leaderboard.html","repository":"https://github.com/NVlabs/RoboLab","task_table":"https://github.com/NVlabs/RoboLab/blob/main/robolab/tasks/README.md"},"robots":[{"id":"franka-robotiq-2f85","name":"Franka + Robotiq 2F-85","body":"arm"}],"body":"arm","bodies":{"arm":53},"instances":53,"task_dirs":106,"modes":{"privileged":53,"standard":53},"oracle":{"full":2,"partial":0,"none":51},"protocols":["eai-standard/2.2"],"budgets_min":[60],"tags":{"perception-understanding":38,"planning-reasoning":33,"control-coordination":53,"manipulation":53,"feedback-adaptation":1},"on_site":true,"preview":[{"task":"block_stacking_specified_order","title":"Block Stacking Specified Order","cover":true,"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/block_stacking_specified_order.jpg?v=c1ca5ce6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/block_stacking_specified_order.mp4?v=f378ff2d","motion":true,"aspect":1.778,"seconds":16},{"task":"put_two_mugs_on_shelf","title":"Put Two Mugs On Shelf","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/put_two_mugs_on_shelf.jpg?v=dd91f0f5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/put_two_mugs_on_shelf.mp4?v=57e3b314","motion":true,"aspect":1.778,"seconds":22},{"task":"cubes_and_blocks_in_bin","title":"Cubes And Blocks In Bin","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/cubes_and_blocks_in_bin.jpg?v=d5736401","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/cubes_and_blocks_in_bin.mp4?v=57d4d867","motion":true,"aspect":1.778,"seconds":18},{"task":"stack3_rubiks_cube","title":"Stack3 Rubiks Cube","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/stack3_rubiks_cube.jpg?v=fcc3467c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/stack3_rubiks_cube.mp4?v=06c493ee","motion":true,"aspect":1.778,"seconds":9},{"task":"unstack_rubiks_cube","title":"Unstack Rubiks Cube","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/unstack_rubiks_cube.jpg?v=65c18d80","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/unstack_rubiks_cube.mp4?v=6b47a250","motion":true,"aspect":1.778,"seconds":9},{"task":"block_stacking_order_agnostic","title":"Block Stacking Order Agnostic","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/block_stacking_order_agnostic.jpg?v=21ade9a8","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/block_stacking_order_agnostic.mp4?v=b00a9272","motion":true,"aspect":1.778,"seconds":9},{"task":"apple_and_yogurt_in_bowl","title":"Apple And Yogurt In Bowl","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/apple_and_yogurt_in_bowl.jpg?v=38f8bac5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/apple_and_yogurt_in_bowl.mp4?v=8bbabf2d","motion":true,"aspect":1.778,"seconds":10},{"task":"banana_then_rubiks_cube","title":"Banana Then Rubiks Cube","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/banana_then_rubiks_cube.jpg?v=36cca786","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/banana_then_rubiks_cube.mp4?v=be5cb2fb","motion":true,"aspect":1.778,"seconds":9}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/_teaser.jpg?v=c1ca5ce6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/_teaser.mp4?v=f378ff2d","cover":true,"aspect":1.778,"seconds":16},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":6,"success":2,"trials":6},"standard":{"done":6,"success":1,"trials":6}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":6,"success":1,"trials":6},"standard":{"done":6,"success":1,"trials":6}},"claude_code-fable_5_2-high":{"privileged":{"done":6,"success":5,"trials":6},"standard":{"done":6,"success":2,"trials":6}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":6,"success":1,"trials":6},"standard":{"done":6,"success":0,"trials":6}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":6,"success":3,"trials":6},"standard":{"done":6,"success":0,"trials":6}},"claude_code-opus_5_5-high":{"privileged":{"done":6,"success":3,"trials":6},"standard":{"done":6,"success":1,"trials":6}},"codex-gpt6_1_sol-high":{"privileged":{"done":6,"success":4,"trials":6},"standard":{"done":6,"success":2,"trials":6}},"codex-gpt6_2_astra-high":{"privileged":{"done":6,"success":6,"trials":6},"standard":{"done":6,"success":2,"trials":6}},"codex-gpt6_astra-high":{"privileged":{"done":6,"success":5,"trials":6},"standard":{"done":6,"success":0,"trials":6}},"codex-gpt6_luna-xhigh":{"privileged":{"done":6,"success":1,"trials":6},"standard":{"done":6,"success":1,"trials":6}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":6,"success":0,"trials":6},"standard":{"done":6,"success":0,"trials":6}}},"solved":{"privileged":43,"standard":28},"attempted":{"privileged":53,"standard":53},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":36.81,"est_n":12,"agent_s":34359,"agent_n":12},"antigravity_cli-gemini_4_argon-high":{"est_usd":29.89,"est_n":12,"agent_s":38776,"agent_n":12},"claude_code-fable_5_2-high":{"est_usd":74.56,"est_n":12,"agent_s":24408,"agent_n":12},"claude_code-glm_5_3_flash-high":{"est_usd":2.81,"est_n":12,"agent_s":38265,"agent_n":12},"claude_code-kimi_3_1_pro-high":{"est_usd":42.22,"est_n":12,"agent_s":40451,"agent_n":12},"claude_code-opus_5_5-high":{"est_usd":22.17,"est_n":12,"agent_s":28590,"agent_n":12},"codex-gpt6_1_sol-high":{"est_usd":26.31,"est_n":12,"agent_s":32013,"agent_n":12},"codex-gpt6_2_astra-high":{"est_usd":46.51,"est_n":12,"agent_s":16817,"agent_n":12},"codex-gpt6_astra-high":{"est_usd":179.87,"est_n":12,"agent_s":35137,"agent_n":12},"codex-gpt6_luna-xhigh":{"est_usd":2.21,"est_n":12,"agent_s":41430,"agent_n":12},"dsh-deepseek_v4_1_flash-high":{"est_usd":3.57,"est_n":12,"agent_s":40944,"agent_n":12}}},{"id":"robocasa","name":"RoboCasa","code":"RCA","origin":"adapted","example":false,"blurb":"Everyday kitchen skills for a mobile manipulator in procedurally generated kitchens.","description":"Everyday kitchen tasks for a mobile manipulator in large procedurally generated kitchens: opening and closing doors and drawers, turning knobs and faucets, pressing buttons, and picking and placing between counters, cabinets, sinks, stoves and appliances, single- and multi-stage. We keep the 29 tasks with an official demonstration at the pinned commit whose demonstration re-verifies as a reference on a frozen instance (its first episode's initial state). Each task passes or fails on the environment's own success checker on the state after the last action, with no partial credit, and runs in two modes: unlimited (privileged: full simulator state, unlimited resets, one replayable trajectory handed in) and limited (standard: the robot's own four cameras, one episode, no reset).","edition":"RoboCasa v0.2 @ 756598a, our selection of 29 tasks (exam @ 8c5594a43)","simulator":"robosuite 1.5.1 / MuJoCo 3.2.6 (RoboCasa v0.2)","links":{"repository":"https://github.com/robocasa/robocasa","paper":"https://arxiv.org/abs/2406.02523","project_page":"https://robocasa.ai"},"robots":[{"id":"franka-panda-omron","name":"Franka Panda on an Omron base","body":"mobile"}],"body":"mobile","bodies":{"mobile":29},"instances":29,"task_dirs":58,"modes":{"privileged":29,"standard":29},"oracle":{"full":29,"partial":0,"none":0},"protocols":["eai-standard/2.1"],"budgets_min":[60],"tags":{"planning-reasoning":5,"control-coordination":29,"manipulation":29,"perception-understanding":1},"on_site":true,"preview":[{"task":"pre_soak_pan","title":"Multi-stage · PreSoakPan","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/pre_soak_pan.jpg?v=a9b27c7a","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/pre_soak_pan.mp4?v=fe02b2fd","motion":true,"aspect":1.778,"seconds":19},{"task":"microwave_thawing","title":"Multi-stage · MicrowaveThawing","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/microwave_thawing.jpg?v=76acb1af","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/microwave_thawing.mp4?v=9d60ffa8","motion":true,"aspect":1.778,"seconds":17},{"task":"prepare_coffee","title":"Multi-stage · PrepareCoffee","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/prepare_coffee.jpg?v=48d01454","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/prepare_coffee.mp4?v=2f4be81d","motion":true,"aspect":1.778,"seconds":19},{"task":"turn_off_microwave","title":"Single-stage · TurnOffMicrowave","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/turn_off_microwave.jpg?v=8f950418","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/turn_off_microwave.mp4?v=8e0741a6","motion":true,"aspect":1.778,"seconds":10},{"task":"coffee_setup_mug","title":"Single-stage · CoffeeSetupMug","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/coffee_setup_mug.jpg?v=adfb9a5c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/coffee_setup_mug.mp4?v=a5b0da9a","motion":true,"aspect":1.778,"seconds":15},{"task":"turn_on_stove","title":"Single-stage · TurnOnStove","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/turn_on_stove.jpg?v=7fe78ade","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/turn_on_stove.mp4?v=a5303bf0","motion":true,"aspect":1.778,"seconds":7},{"task":"open_double_door","title":"Single-stage · OpenDoubleDoor","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/open_double_door.jpg?v=e6f10c87","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/open_double_door.mp4?v=39202d74","motion":true,"aspect":1.778,"seconds":17},{"task":"arrange_vegetables","title":"Multi-stage · ArrangeVegetables","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/arrange_vegetables.jpg?v=67364597","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/arrange_vegetables.mp4?v=e1616a93","motion":true,"aspect":1.778,"seconds":16}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/_teaser.jpg?v=e3958544","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/_teaser.mp4?v=31f6b494","cover":true,"aspect":1.778,"seconds":36},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"claude_code-fable_5_2-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"claude_code-opus_5_5-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_1_sol-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_2_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"codex-gpt6_luna-xhigh":{"privileged":{"done":2,"success":1,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}}},"solved":{"privileged":29,"standard":10},"attempted":{"privileged":29,"standard":29},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":9.69,"est_n":4,"agent_s":6381,"agent_n":4},"antigravity_cli-gemini_4_argon-high":{"est_usd":11.23,"est_n":4,"agent_s":8934,"agent_n":4},"claude_code-fable_5_2-high":{"est_usd":12.11,"est_n":4,"agent_s":2765,"agent_n":4},"claude_code-glm_5_3_flash-high":{"est_usd":1.48,"est_n":4,"agent_s":10120,"agent_n":4},"claude_code-kimi_3_1_pro-high":{"est_usd":10.8,"est_n":4,"agent_s":8113,"agent_n":4},"claude_code-opus_5_5-high":{"est_usd":12.22,"est_n":4,"agent_s":4162,"agent_n":4},"codex-gpt6_1_sol-high":{"est_usd":2.62,"est_n":4,"agent_s":3645,"agent_n":4},"codex-gpt6_2_astra-high":{"est_usd":3.68,"est_n":4,"agent_s":1064,"agent_n":4},"codex-gpt6_astra-high":{"est_usd":89.41,"est_n":4,"agent_s":12967,"agent_n":4},"codex-gpt6_luna-xhigh":{"est_usd":0.7,"est_n":4,"agent_s":10051,"agent_n":4},"dsh-deepseek_v4_1_flash-high":{"est_usd":0.95,"est_n":4,"agent_s":9215,"agent_n":4}}},{"id":"robocasa365","name":"RoboCasa365","code":"R365","origin":"adapted","example":false,"blurb":"Atomic and long composite kitchen activities from RoboCasa's 365-task suite.","description":"RoboCasa's 365-task kitchen suite for the same mobile manipulator: atomic skills and long composite activities across cooking, cleaning, storing and serving. We keep 50 of the families with a validated demonstration, chosen by a fixed rule (12 atomic, 38 composite, in SHA-256 order of their names), one frozen instance each. Each task passes or fails on the environment's own success checker on the state after the last action, with no partial credit, and runs in two modes: unlimited (privileged: full simulator state, unlimited resets, one replayable trajectory handed in) and limited (standard: the robot's own four cameras, one episode, no reset).","edition":"RoboCasa365 (robocasa 1.0.1 @ 4f8a298), our selection of 50 tasks (exam @ 8c5594a43)","simulator":"robosuite 1.5.2 / MuJoCo 3.3.1 (RoboCasa365)","links":{"repository":"https://github.com/robocasa/robocasa","project":"https://robocasa.ai","paper":"https://arxiv.org/abs/2603.04356","project_page":"https://robocasa.ai"},"robots":[{"id":"franka-panda-omron","name":"Franka Panda on an Omron base","body":"mobile"}],"body":"mobile","bodies":{"mobile":50},"instances":50,"task_dirs":101,"modes":{"privileged":50,"standard":50,"realtime":1},"oracle":{"full":50,"partial":0,"none":0},"protocols":["eai-standard/2.1","eai-standard/2.3"],"budgets_min":[60],"tags":{"control-coordination":50,"manipulation":28,"planning-reasoning":38,"mobile-whole-body-manipulation":22,"perception-understanding":2},"on_site":true,"preview":[{"task":"seasoning_spice_setup","title":"Composite · SeasoningSpiceSetup","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/seasoning_spice_setup.jpg?v=6a9238c0","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/seasoning_spice_setup.mp4?v=4e19e794","motion":true,"aspect":1.778,"seconds":26},{"task":"searing_meat","title":"Composite · SearingMeat","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/searing_meat.jpg?v=055df49e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/searing_meat.mp4?v=5c11e3df","motion":true,"aspect":1.778,"seconds":18},{"task":"make_ice_lemonade","title":"Composite · MakeIceLemonade","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/make_ice_lemonade.jpg?v=3131661b","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/make_ice_lemonade.mp4?v=3b80363b","motion":true,"aspect":1.778,"seconds":22},{"task":"reheat_meal","title":"Composite · ReheatMeal","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/reheat_meal.jpg?v=efb3e58b","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/reheat_meal.mp4?v=1aed02be","motion":true,"aspect":1.778,"seconds":17},{"task":"cereal_and_bowl","title":"Composite · CerealAndBowl","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/cereal_and_bowl.jpg?v=92236424","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/cereal_and_bowl.mp4?v=6946185b","motion":true,"aspect":1.778,"seconds":22},{"task":"microwave_defrost_meat","title":"Composite · MicrowaveDefrostMeat","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/microwave_defrost_meat.jpg?v=8eeb1878","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/microwave_defrost_meat.mp4?v=643add4f","motion":true,"aspect":1.778,"seconds":20},{"task":"retrieve_meat","title":"Composite · RetrieveMeat","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/retrieve_meat.jpg?v=0a94d5f4","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/retrieve_meat.mp4?v=61ee7b71","motion":true,"aspect":1.778,"seconds":16},{"task":"deliver_brewed_coffee","title":"Composite · DeliverBrewedCoffee","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/deliver_brewed_coffee.jpg?v=f09ad325","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/deliver_brewed_coffee.mp4?v=61d029bf","motion":true,"aspect":1.778,"seconds":17}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/_teaser.jpg?v=a1bf1d3a","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/_teaser.mp4?v=f31f8a70","cover":true,"aspect":1.778,"seconds":34},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":5,"success":0,"trials":5},"standard":{"done":5,"success":0,"trials":5}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":5,"success":3,"trials":5},"standard":{"done":5,"success":0,"trials":5}},"claude_code-fable_5_2-high":{"privileged":{"done":5,"success":5,"trials":5},"standard":{"done":5,"success":2,"trials":5}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":5,"success":0,"trials":5},"standard":{"done":5,"success":0,"trials":5}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":5,"success":0,"trials":5},"standard":{"done":5,"success":0,"trials":5}},"claude_code-opus_5_5-high":{"privileged":{"done":5,"success":4,"trials":5},"standard":{"done":5,"success":1,"trials":5}},"codex-gpt6_1_sol-high":{"privileged":{"done":5,"success":5,"trials":5},"standard":{"done":5,"success":1,"trials":5}},"codex-gpt6_2_astra-high":{"privileged":{"done":5,"success":5,"trials":5},"standard":{"done":5,"success":3,"trials":5}},"codex-gpt6_astra-high":{"privileged":{"done":5,"success":5,"trials":5},"standard":{"done":5,"success":1,"trials":5}},"codex-gpt6_luna-xhigh":{"privileged":{"done":5,"success":0,"trials":5},"standard":{"done":5,"success":0,"trials":5}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":5,"success":0,"trials":5},"standard":{"done":5,"success":0,"trials":5}}},"solved":{"privileged":50,"standard":23},"attempted":{"privileged":50,"standard":50},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":73.01,"est_n":10,"agent_s":31928,"agent_n":10},"antigravity_cli-gemini_4_argon-high":{"est_usd":44.35,"est_n":10,"agent_s":30658,"agent_n":10},"claude_code-fable_5_2-high":{"est_usd":77.78,"est_n":10,"agent_s":18231,"agent_n":10},"claude_code-glm_5_3_flash-high":{"est_usd":4.59,"est_n":10,"agent_s":28399,"agent_n":10},"claude_code-kimi_3_1_pro-high":{"est_usd":50.06,"est_n":10,"agent_s":35007,"agent_n":10},"claude_code-opus_5_5-high":{"est_usd":41.08,"est_n":10,"agent_s":23283,"agent_n":10},"codex-gpt6_1_sol-high":{"est_usd":20.3,"est_n":10,"agent_s":22774,"agent_n":10},"codex-gpt6_2_astra-high":{"est_usd":50.62,"est_n":10,"agent_s":12039,"agent_n":10},"codex-gpt6_astra-high":{"est_usd":150.41,"est_n":10,"agent_s":22842,"agent_n":10},"codex-gpt6_luna-xhigh":{"est_usd":4.23,"est_n":10,"agent_s":31959,"agent_n":10},"dsh-deepseek_v4_1_flash-high":{"est_usd":2.28,"est_n":10,"agent_s":27865,"agent_n":10}}},{"id":"robocasa-gr1","name":"RoboCasa-GR1","code":"RGR1","origin":"adapted","example":false,"blurb":"A Fourier GR1 humanoid with dexterous hands, picking and placing at a counter.","description":"A Fourier GR1 humanoid with two dexterous hands at a kitchen counter: picking objects from one container or surface and placing them in another, including novel objects, and placing into a cabinet, drawer or microwave and closing it. We keep all 24 environments with an official demonstration, one frozen instance each (the demonstration's initial state). Each task passes or fails on the environment's own success checker on the state after the last action, with no partial credit, and runs in two modes: unlimited (privileged: full simulator state, unlimited resets, one replayable trajectory handed in) and limited (standard: the humanoid's own six cameras, one episode, no reset).","edition":"RoboCasa GR1 tabletop tasks @ 4840e67, all 24 (exam @ 8c5594a43)","simulator":"robosuite @ 51cc017 (1.5.1, mink whole-body IK) / MuJoCo 3.2.6 (RoboCasa GR1 tabletop)","links":{"repository":"https://github.com/robocasa/robocasa-gr1-tabletop-tasks","paper":"https://arxiv.org/abs/2503.14734","project_page":"https://developer.nvidia.com/isaac/gr00t"},"robots":[{"id":"fourier-gr1","name":"Fourier GR1","body":"humanoid"}],"body":"humanoid","bodies":{"humanoid":24},"instances":24,"task_dirs":48,"modes":{"privileged":24,"standard":24},"oracle":{"full":24,"partial":0,"none":0},"protocols":["eai-standard/2.1"],"budgets_min":[60],"tags":{"planning-reasoning":6,"control-coordination":24,"manipulation":24},"on_site":true,"preview":[{"task":"potato_to_microwave_close","title":"GR1 · PnPPotatoToMicrowaveClose","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/potato_to_microwave_close.jpg?v=e020e0c9","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/potato_to_microwave_close.mp4?v=770087de","motion":true,"aspect":1.778,"seconds":13},{"task":"wine_to_cabinet_close","title":"GR1 · PnPWineToCabinetClose","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/wine_to_cabinet_close.jpg?v=6fe5e6f6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/wine_to_cabinet_close.mp4?v=3a54678a","motion":true,"aspect":1.778,"seconds":15},{"task":"plate_to_bowl","title":"GR1 · PosttrainPnPNovelFromPlateToBowlSplitA","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/plate_to_bowl.jpg?v=7b6dd1bf","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/plate_to_bowl.mp4?v=73ce4a58","motion":true,"aspect":1.778,"seconds":9},{"task":"cup_to_drawer_close","title":"GR1 · PnPCupToDrawerClose","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/cup_to_drawer_close.jpg?v=d0d81c28","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/cup_to_drawer_close.mp4?v=d905657d","motion":true,"aspect":1.778,"seconds":12},{"task":"tray_to_tieredshelf","title":"GR1 · PosttrainPnPNovelFromTrayToTieredshelfSplitA","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/tray_to_tieredshelf.jpg?v=5a9c9b4b","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/tray_to_tieredshelf.mp4?v=0e98d715","motion":true,"aspect":1.778,"seconds":11},{"task":"plate_to_cardboardbox","title":"GR1 · PosttrainPnPNovelFromPlateToCardboardboxSplitA","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/plate_to_cardboardbox.jpg?v=8fd5a306","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/plate_to_cardboardbox.mp4?v=3dd60700","motion":true,"aspect":1.778,"seconds":11},{"task":"placemat_to_tieredshelf","title":"GR1 · PosttrainPnPNovelFromPlacematToTieredshelfSplitA","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/placemat_to_tieredshelf.jpg?v=aec5f462","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/placemat_to_tieredshelf.mp4?v=a3e424e1","motion":true,"aspect":1.778,"seconds":10},{"task":"bottle_to_cabinet_close","title":"GR1 · PnPBottleToCabinetClose","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/bottle_to_cabinet_close.jpg?v=54d1349a","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/bottle_to_cabinet_close.mp4?v=69bd8544","motion":true,"aspect":1.778,"seconds":14}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/_teaser.jpg?v=e020e0c9","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/_teaser.mp4?v=770087de","cover":true,"aspect":1.778,"seconds":13},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":1,"success":0,"trials":1},"standard":{"done":1,"success":0,"trials":1}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":1,"success":1,"trials":1},"standard":{"done":1,"success":0,"trials":1}},"claude_code-fable_5_2-high":{"privileged":{"done":1,"success":1,"trials":1},"standard":{"done":1,"success":1,"trials":1}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":1,"success":0,"trials":1},"standard":{"done":1,"success":0,"trials":1}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":1,"success":0,"trials":1},"standard":{"done":1,"success":1,"trials":1}},"claude_code-opus_5_5-high":{"privileged":{"done":1,"success":1,"trials":1},"standard":{"done":1,"success":1,"trials":1}},"codex-gpt6_1_sol-high":{"privileged":{"done":1,"success":1,"trials":1},"standard":{"done":1,"success":0,"trials":1}},"codex-gpt6_2_astra-high":{"privileged":{"done":1,"success":1,"trials":1},"standard":{"done":1,"success":1,"trials":1}},"codex-gpt6_astra-high":{"privileged":{"done":1,"success":1,"trials":1},"standard":{"done":1,"success":0,"trials":1}},"codex-gpt6_luna-xhigh":{"privileged":{"done":1,"success":1,"trials":1},"standard":{"done":1,"success":0,"trials":1}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":1,"success":0,"trials":1},"standard":{"done":1,"success":0,"trials":1}}},"solved":{"privileged":23,"standard":11},"attempted":{"privileged":24,"standard":24},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":7.15,"est_n":2,"agent_s":5049,"agent_n":2},"antigravity_cli-gemini_4_argon-high":{"est_usd":7.11,"est_n":2,"agent_s":5544,"agent_n":2},"claude_code-fable_5_2-high":{"est_usd":16.43,"est_n":2,"agent_s":4295,"agent_n":2},"claude_code-glm_5_3_flash-high":{"est_usd":0.94,"est_n":2,"agent_s":6600,"agent_n":2},"claude_code-kimi_3_1_pro-high":{"est_usd":6.3,"est_n":2,"agent_s":5327,"agent_n":2},"claude_code-opus_5_5-high":{"est_usd":6.92,"est_n":2,"agent_s":4222,"agent_n":2},"codex-gpt6_1_sol-high":{"est_usd":3.1,"est_n":2,"agent_s":4703,"agent_n":2},"codex-gpt6_2_astra-high":{"est_usd":3.87,"est_n":2,"agent_s":1122,"agent_n":2},"codex-gpt6_astra-high":{"est_usd":30.46,"est_n":2,"agent_s":4599,"agent_n":2},"codex-gpt6_luna-xhigh":{"est_usd":0.96,"est_n":2,"agent_s":5700,"agent_n":2},"dsh-deepseek_v4_1_flash-high":{"est_usd":0.4,"est_n":2,"agent_s":5731,"agent_n":2}}},{"id":"metaworldplus","name":"MetaWorld+","code":"MW+","origin":"adapted","example":false,"blurb":"All 50 Meta-World v3 environments for a Sawyer arm, one frozen instance each.","description":"All 50 of Meta-World's v3 environments for a Sawyer arm (reward v2), one frozen instance each: push, pick and place, open doors, drawers and windows, press buttons, turn dials and faucets, insert pegs, hammer, sweep and more. Each task passes or fails on Meta-World's own success checker on the state after the agent's last action, with no partial credit, and runs in two modes: unlimited (privileged: full simulator state, unlimited resets, one replayable trajectory of at most 500 actions handed in) and limited (standard: a robot behind a service, four fixed cameras and the arm's own state, one episode, no reset). Privileged is reported as a control; the standard mode is the benchmark.","edition":"our benchmark (exam @ 8c5594a43): Meta-World v3 @ 59fc34d, all 50 environments","simulator":"MuJoCo 3.3.0 (Meta-World v3)","links":{"repository":"https://github.com/Farama-Foundation/Metaworld","paper":"https://arxiv.org/abs/2505.11289","project_page":"https://metaworld.farama.org/"},"robots":[{"id":"sawyer","name":"Rethink Sawyer","body":"arm"}],"body":"arm","bodies":{"arm":50},"instances":50,"task_dirs":100,"modes":{"privileged":50,"standard":50},"oracle":{"full":50,"partial":0,"none":0},"protocols":["eai-standard/2.1"],"budgets_min":[30],"tags":{"control-coordination":50,"manipulation":50,"planning-reasoning":8},"on_site":true,"preview":[{"task":"disassemble","title":"Meta-World · Disassemble","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/disassemble.jpg?v=cbc0ee53","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/disassemble.mp4?v=c100921e","motion":true,"aspect":1.778,"seconds":7},{"task":"basketball","title":"Meta-World · Basketball","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/basketball.jpg?v=85b4dcb2","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/basketball.mp4?v=d092027e","motion":true,"aspect":1.778,"seconds":5},{"task":"stick_pull","title":"Meta-World · Stick pull","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/stick_pull.jpg?v=377c1c75","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/stick_pull.mp4?v=5898316c","motion":true,"aspect":1.778,"seconds":6},{"task":"peg_insert_side","title":"Meta-World · Peg insert side","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/peg_insert_side.jpg?v=649fdbb2","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/peg_insert_side.mp4?v=8b03372c","motion":true,"aspect":1.778,"seconds":7},{"task":"handle_pull_side","title":"Meta-World · Handle pull side","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/handle_pull_side.jpg?v=d9f9f5a7","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/handle_pull_side.mp4?v=9f4f2a29","motion":true,"aspect":1.778,"seconds":7},{"task":"stick_push","title":"Meta-World · Stick push","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/stick_push.jpg?v=778b2193","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/stick_push.mp4?v=f9fe2aec","motion":true,"aspect":1.778,"seconds":7},{"task":"hammer","title":"Meta-World · Hammer","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/hammer.jpg?v=2c433240","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/hammer.mp4?v=dacd5adf","motion":true,"aspect":1.778,"seconds":7},{"task":"assembly","title":"Meta-World · Assembly","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/assembly.jpg?v=9c87de1a","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/assembly.mp4?v=e761553b","motion":true,"aspect":1.778,"seconds":5}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/_teaser.jpg?v=85d56c4b","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/_teaser.mp4?v=147e7847","cover":true,"aspect":1.778,"seconds":15},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"claude_code-fable_5_2-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"claude_code-opus_5_5-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_1_sol-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_2_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"codex-gpt6_luna-xhigh":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}}},"solved":{"privileged":50,"standard":31},"attempted":{"privileged":50,"standard":50},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":4.13,"est_n":4,"agent_s":1653,"agent_n":4},"antigravity_cli-gemini_4_argon-high":{"est_usd":2.53,"est_n":4,"agent_s":2171,"agent_n":4},"claude_code-fable_5_2-high":{"est_usd":3.96,"est_n":4,"agent_s":905,"agent_n":4},"claude_code-glm_5_3_flash-high":{"est_usd":0.74,"est_n":4,"agent_s":3832,"agent_n":4},"claude_code-kimi_3_1_pro-high":{"est_usd":1.89,"est_n":4,"agent_s":1147,"agent_n":4},"claude_code-opus_5_5-high":{"est_usd":2.08,"est_n":4,"agent_s":1345,"agent_n":4},"codex-gpt6_1_sol-high":{"est_usd":0.68,"est_n":4,"agent_s":957,"agent_n":4},"codex-gpt6_2_astra-high":{"est_usd":1.35,"est_n":4,"agent_s":433,"agent_n":4},"codex-gpt6_astra-high":{"est_usd":4.8,"est_n":4,"agent_s":905,"agent_n":4},"codex-gpt6_luna-xhigh":{"est_usd":0.33,"est_n":4,"agent_s":2291,"agent_n":4},"dsh-deepseek_v4_1_flash-high":{"est_usd":0.57,"est_n":4,"agent_s":3418,"agent_n":4}}},{"id":"vlabench","name":"VLABench","code":"VLB","origin":"adapted","example":false,"blurb":"Manipulation from language: attributes, common sense and multi-step chores.","description":"Language-conditioned manipulation for a Franka Panda at a table: selecting by attribute, common sense, semantic and spatial descriptions, physical question answering by pressing buttons, and multi-step tasks (heating food, setting a table, playing poker). We keep the 36 of VLABench's native tasks whose recorded expert re-verifies on a frozen instance, one instance each. Each task passes or fails on VLABench's own success checker (its termination condition), with no partial credit, and runs in two modes: unlimited (privileged: every entity's pose, VLABench's skills as planning primitives, unlimited resets, one replayable trajectory handed in) and limited (standard: four cameras and the arm's own state, one episode, no reset).","edition":"VLABench @ cf588fe, our selection of 36 tasks (exam @ 8c5594a43)","simulator":"MuJoCo 3.2.2 + dm_control 1.0.22 (VLABench)","links":{"repository":"https://github.com/OpenMOSS/VLABench","paper":"https://arxiv.org/abs/2412.18194","project_page":"https://vlabench.github.io/"},"robots":[{"id":"franka-panda","name":"Franka Panda","body":"arm"}],"body":"arm","bodies":{"arm":36},"instances":36,"task_dirs":72,"modes":{"privileged":36,"standard":36},"oracle":{"full":36,"partial":0,"none":0},"protocols":["eai-standard/2.1"],"budgets_min":[30],"tags":{"perception-understanding":31,"planning-reasoning":19,"control-coordination":36,"manipulation":36,"feedback-adaptation":1},"on_site":true,"preview":[{"task":"heat_food","title":"Manipulation · heat_food","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/heat_food.jpg?v=7af38c99","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/heat_food.mp4?v=5ce5ea4f","motion":true,"aspect":1.778,"seconds":22},{"task":"texas_holdem","title":"Manipulation · texas_holdem","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/texas_holdem.jpg?v=72bb5c84","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/texas_holdem.mp4?v=cc9b2357","motion":true,"aspect":1.778,"seconds":19},{"task":"take_out_cool_drink","title":"Manipulation · take_out_cool_drink","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/take_out_cool_drink.jpg?v=23cc5910","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/take_out_cool_drink.mp4?v=f12b5897","motion":true,"aspect":1.778,"seconds":19},{"task":"hammer_nail_and_hang_picture","title":"Manipulation · hammer_nail_and_hang_picture","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/hammer_nail_and_hang_picture.jpg?v=ce7c5b3c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/hammer_nail_and_hang_picture.mp4?v=4286508f","motion":true,"aspect":1.778,"seconds":17},{"task":"book_rearrange","title":"Manipulation · book_rearrange","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/book_rearrange.jpg?v=984e5ba7","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/book_rearrange.mp4?v=a3048093","motion":true,"aspect":1.778,"seconds":18},{"task":"select_unique_type_mahjong","title":"Manipulation · select_unique_type_mahjong","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/select_unique_type_mahjong.jpg?v=c7201f9d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/select_unique_type_mahjong.mp4?v=fe96f7c2","motion":true,"aspect":1.778,"seconds":13},{"task":"insert_flower","title":"Manipulation · insert_flower","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/insert_flower.jpg?v=d28643e2","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/insert_flower.mp4?v=44396210","motion":true,"aspect":1.778,"seconds":18},{"task":"select_billiards_common_sense","title":"Common sense · select_billiards_common_sense","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/select_billiards_common_sense.jpg?v=b86f50fa","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/select_billiards_common_sense.mp4?v=704b3aba","motion":true,"aspect":1.778,"seconds":15}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/_teaser.jpg?v=4775d7d8","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/_teaser.mp4?v=9c3564ee","cover":true,"aspect":1.778,"seconds":38},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":2,"success":0,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":2,"success":1,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"claude_code-fable_5_2-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":2,"success":0,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":2,"success":1,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"claude_code-opus_5_5-high":{"privileged":{"done":2,"success":1,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"codex-gpt6_1_sol-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"codex-gpt6_2_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"codex-gpt6_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"codex-gpt6_luna-xhigh":{"privileged":{"done":2,"success":1,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":2,"success":1,"trials":2},"standard":{"done":2,"success":0,"trials":2}}},"solved":{"privileged":35,"standard":16},"attempted":{"privileged":36,"standard":36},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":27.98,"est_n":4,"agent_s":11105,"agent_n":4},"antigravity_cli-gemini_4_argon-high":{"est_usd":6.25,"est_n":4,"agent_s":7130,"agent_n":4},"claude_code-fable_5_2-high":{"est_usd":15.24,"est_n":4,"agent_s":3526,"agent_n":4},"claude_code-glm_5_3_flash-high":{"est_usd":1.12,"est_n":4,"agent_s":11125,"agent_n":4},"claude_code-kimi_3_1_pro-high":{"est_usd":12.59,"est_n":4,"agent_s":6750,"agent_n":4},"claude_code-opus_5_5-high":{"est_usd":11.99,"est_n":4,"agent_s":5404,"agent_n":4},"codex-gpt6_1_sol-high":{"est_usd":3.97,"est_n":4,"agent_s":5380,"agent_n":4},"codex-gpt6_2_astra-high":{"est_usd":8.99,"est_n":4,"agent_s":3069,"agent_n":4},"codex-gpt6_astra-high":{"est_usd":51.02,"est_n":4,"agent_s":10015,"agent_n":4},"codex-gpt6_luna-xhigh":{"est_usd":0.8,"est_n":4,"agent_s":11793,"agent_n":4},"dsh-deepseek_v4_1_flash-high":{"est_usd":1.12,"est_n":4,"agent_s":10034,"agent_n":4}}},{"id":"humanoidbench","name":"HumanoidBench","code":"HB","origin":"adapted","example":false,"blurb":"Whole-body locomotion and manipulation for a Unitree H1 humanoid.","description":"Whole-body tasks for a simulated Unitree H1 humanoid, bare or with two Shadow dexterous hands: locomotion (walking, stairs, hurdles, balancing, a maze, sitting) and whole-body manipulation (doors, cabinets, a bookshelf, pushing and lifting, a cube in each hand). Each episode is scored by HumanoidBench's own dense reward, summed over the episode, against its own success bar. We keep the 9 hardest of its 32 registered tasks plus one fixture, in three modes: privileged (full simulator state, unlimited resets, one replayable trajectory handed in), standard (a robot behind a service: cameras, joint readings and an IMU, 50 resets) and realtime (standard on a clock that does not stop: the robot runs at 50 Hz while the agent thinks).","edition":"cb11890 (upstream), our selection of 9 tasks and a fixture (2026-10-04; 24 until then)","simulator":"MuJoCo 3 (HumanoidBench's gymnasium environments)","links":{"paper":"https://arxiv.org/abs/2403.10506","project_page":"https://humanoid-bench.github.io","repository":"https://github.com/carlosferrazza/humanoid-bench"},"robots":[{"id":"unitree-h1","name":"Unitree H1","body":"humanoid"},{"id":"unitree-h1hand","name":"Unitree H1 with hands","body":"humanoid"}],"body":"humanoid","bodies":{"humanoid":10},"instances":10,"task_dirs":30,"modes":{"privileged":10,"standard":10,"realtime":10},"oracle":{"full":1,"partial":0,"none":9},"protocols":["eai-standard/2.1","eai-standard/2.3"],"budgets_min":[60],"tags":{"control-coordination":10,"feedback-adaptation":2,"locomotion-stability":4,"planning-reasoning":4,"mobile-whole-body-manipulation":6,"navigation-exploration":1},"on_site":true,"preview":[{"task":"manip_cube","title":"Manipulation · Cube","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/manip_cube.jpg?v=260e13ab","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/manip_cube.mp4?v=76421663","motion":true,"aspect":1.778,"seconds":11},{"task":"loco_balance_hard","title":"Locomotion · Balance hard","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/loco_balance_hard.jpg?v=71b6721d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/loco_balance_hard.mp4?v=e1c7fb8e","motion":true,"aspect":1.778,"seconds":22},{"task":"manip_window","title":"Manipulation · Window","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/manip_window.jpg?v=b6a3ffee","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/manip_window.mp4?v=ffe49526","motion":true,"aspect":1.778,"seconds":22},{"task":"loco_hurdle","title":"Locomotion · Hurdle","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/loco_hurdle.jpg?v=066370cd","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/loco_hurdle.mp4?v=9af19d00","motion":true,"aspect":1.778,"seconds":22},{"task":"loco_walk","title":"Locomotion · Walk","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/loco_walk.jpg?v=4b5a761a","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/loco_walk.mp4?v=ac205a0d","motion":true,"aspect":1.778,"seconds":20},{"task":"loco_sit_simple","title":"Locomotion · Sit simple","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/loco_sit_simple.jpg?v=64577c45","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/loco_sit_simple.mp4?v=56930922","motion":true,"aspect":1.778,"seconds":20},{"task":"manip_bookshelf_simple","title":"Manipulation · Bookshelf simple","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/manip_bookshelf_simple.jpg?v=388083f5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/manip_bookshelf_simple.mp4?v=fca264ec","motion":false,"aspect":1.778,"seconds":7},{"task":"manip_door","title":"Manipulation · Door","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/manip_door.jpg?v=992050cf","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/manip_door.mp4?v=b05b3e1a","motion":false,"aspect":1.778,"seconds":7}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/_teaser.jpg?v=260e13ab","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/_teaser.mp4?v=76421663","cover":true,"aspect":1.778,"seconds":11},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":8,"success":0,"trials":8},"standard":{"done":16,"success":0,"trials":16}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":8,"success":2,"trials":8},"standard":{"done":16,"success":0,"trials":16}},"claude_code-fable_5_2-high":{"privileged":{"done":8,"success":3,"trials":8},"standard":{"done":16,"success":1,"trials":16}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":8,"success":0,"trials":8},"standard":{"done":16,"success":0,"trials":16}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":8,"success":1,"trials":8},"standard":{"done":16,"success":0,"trials":16}},"claude_code-opus_5_5-high":{"privileged":{"done":8,"success":2,"trials":8},"standard":{"done":16,"success":0,"trials":16}},"codex-gpt6_1_sol-high":{"privileged":{"done":8,"success":3,"trials":8},"standard":{"done":16,"success":0,"trials":16}},"codex-gpt6_2_astra-high":{"privileged":{"done":8,"success":5,"trials":8},"standard":{"done":16,"success":2,"trials":16}},"codex-gpt6_astra-high":{"privileged":{"done":8,"success":3,"trials":8},"standard":{"done":16,"success":2,"trials":16}},"codex-gpt6_luna-xhigh":{"privileged":{"done":8,"success":0,"trials":8},"standard":{"done":16,"success":0,"trials":16}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":8,"success":0,"trials":8},"standard":{"done":16,"success":0,"trials":16}}},"solved":{"privileged":7,"standard":3},"attempted":{"privileged":10,"standard":10},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":128.4,"est_n":24,"agent_s":54593,"agent_n":24},"antigravity_cli-gemini_4_argon-high":{"est_usd":108.31,"est_n":24,"agent_s":86019,"agent_n":24},"claude_code-fable_5_2-high":{"est_usd":272.19,"est_n":24,"agent_s":50112,"agent_n":18},"claude_code-glm_5_3_flash-high":{"est_usd":8.5,"est_n":24,"agent_s":41617,"agent_n":24},"claude_code-kimi_3_1_pro-high":{"est_usd":133.67,"est_n":24,"agent_s":66253,"agent_n":24},"claude_code-opus_5_5-high":{"est_usd":76.74,"est_n":24,"agent_s":36607,"agent_n":24},"codex-gpt6_1_sol-high":{"est_usd":65.84,"est_n":24,"agent_s":86224,"agent_n":24},"codex-gpt6_2_astra-high":{"est_usd":262.39,"est_n":24,"agent_s":81090,"agent_n":24},"codex-gpt6_astra-high":{"est_usd":546.31,"est_n":24,"agent_s":81844,"agent_n":24},"codex-gpt6_luna-xhigh":{"est_usd":9.17,"est_n":24,"agent_s":64223,"agent_n":24},"dsh-deepseek_v4_1_flash-high":{"est_usd":6.18,"est_n":24,"agent_s":46127,"agent_n":24}}},{"id":"molmospaces","name":"MolmoSpaces","code":"MOL","origin":"adapted","example":false,"blurb":"Zero-shot episodes in unseen houses, for a Franka FR3 and an RB-Y1 mobile robot.","description":"Ai2's zero-shot evaluation episodes for robot policies in unseen houses: a Franka FR3 picks, places (into a receptacle, next to an object, by colour), opens and closes fixtures, and a Rainbow RB-Y1 mobile manipulator navigates to an object and opens a room door. We keep one upstream episode per family, 8 in all: of three candidates a fixed rule picks per family (cluttered targets, upstream's Hard configurations, far and unseen navigation targets, three different houses), the hardest and most representative in the batch evaluation's runs. Each task passes or fails on upstream's own `judge_success()` at the end of the episode, with no partial credit, and runs in two modes: unlimited (privileged: the simulator's full state, upstream's kinematics, unlimited resets, one replayable trajectory handed in) and limited (standard: the robot's own cameras with depth and its proprioception, one episode, no reset).","edition":"molmo-spaces 0.2.9, benchmark molmospaces-bench-v2/20260415; our selection of 8 episodes (exam @ e46be110c,","simulator":"MuJoCo 3.5.0 (molmo-spaces 0.2.9)","links":{"repository":"https://github.com/allenai/molmospaces","paper":"https://arxiv.org/abs/2602.11337","molmobot_paper":"https://arxiv.org/abs/2603.16861","project_page":"https://allenai.github.io/molmospaces/"},"robots":[{"id":"franka-robotiq-2f85","name":"Franka + Robotiq 2F-85","body":"arm"},{"id":"rainbow-rby1","name":"Rainbow RB-Y1","body":"mobile"}],"body":"arm","bodies":{"arm":6,"mobile":2},"instances":8,"task_dirs":16,"modes":{"privileged":8,"standard":8},"oracle":{"full":0,"partial":0,"none":8},"protocols":["eai-standard/2.2"],"budgets_min":[60],"tags":{"control-coordination":8,"manipulation":6,"perception-understanding":3,"feedback-adaptation":1,"navigation-exploration":1,"mobile-whole-body-manipulation":1},"on_site":true,"preview":[{"task":"pick_and_place_color","title":"Franka · Pick and place by colour","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/pick_and_place_color.jpg?v=b226ab56","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/pick_and_place_color.mp4?v=7c3421ce","motion":true,"aspect":1.778,"seconds":22},{"task":"open","title":"Franka · Open","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/open.jpg?v=962a5ced","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/open.mp4?v=8e8e2511","motion":true,"aspect":1.778,"seconds":6},{"task":"pick_and_place_next_to","title":"Franka · Pick and place next to","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/pick_and_place_next_to.jpg?v=05eb8714","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/pick_and_place_next_to.mp4?v=11bb05b7","motion":true,"aspect":1.778,"seconds":14},{"task":"pick","title":"Franka · Pick","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/pick.jpg?v=741d138e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/pick.mp4?v=643bb113","motion":true,"aspect":1.778,"seconds":9},{"task":"close","title":"Franka · Close","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/close.jpg?v=95d34a83","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/close.mp4?v=184ae756","motion":true,"aspect":1.778,"seconds":6},{"task":"pick_and_place","title":"Franka · Pick and place","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/pick_and_place.jpg?v=688d1704","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/pick_and_place.mp4?v=bdd7ce32","motion":true,"aspect":1.778,"seconds":11},{"task":"navigate_to","title":"RB-Y1 · Navigate to","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/navigate_to.jpg?v=7c95f3fc","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/navigate_to.mp4?v=ee6bfb29","motion":true,"aspect":1.778,"seconds":21},{"task":"open_door","title":"RB-Y1 · Open door","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/open_door.jpg?v=a3ca097c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/open_door.mp4?v=02d59274","motion":true,"aspect":1.778,"seconds":6}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/_teaser.jpg?v=b226ab56","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/_teaser.mp4?v=7c3421ce","cover":true,"aspect":1.778,"seconds":22},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"claude_code-fable_5_2-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"claude_code-opus_5_5-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_1_sol-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_2_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_luna-xhigh":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}}},"solved":{"privileged":7,"standard":6},"attempted":{"privileged":8,"standard":8},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":4.76,"est_n":4,"agent_s":2305,"agent_n":4},"antigravity_cli-gemini_4_argon-high":{"est_usd":4.9,"est_n":4,"agent_s":4908,"agent_n":4},"claude_code-fable_5_2-high":{"est_usd":11.85,"est_n":4,"agent_s":1934,"agent_n":4},"claude_code-glm_5_3_flash-high":{"est_usd":0.67,"est_n":4,"agent_s":3451,"agent_n":4},"claude_code-kimi_3_1_pro-high":{"est_usd":3.29,"est_n":4,"agent_s":1677,"agent_n":4},"claude_code-opus_5_5-high":{"est_usd":4.33,"est_n":4,"agent_s":2117,"agent_n":4},"codex-gpt6_1_sol-high":{"est_usd":1.77,"est_n":4,"agent_s":1887,"agent_n":4},"codex-gpt6_2_astra-high":{"est_usd":2.86,"est_n":4,"agent_s":912,"agent_n":4},"codex-gpt6_astra-high":{"est_usd":9.37,"est_n":4,"agent_s":1870,"agent_n":4},"codex-gpt6_luna-xhigh":{"est_usd":0.38,"est_n":4,"agent_s":3885,"agent_n":4},"dsh-deepseek_v4_1_flash-high":{"est_usd":0.99,"est_n":4,"agent_s":8753,"agent_n":4}}},{"id":"kinder","name":"KinDER","code":"KND","origin":"adapted","example":false,"blurb":"Physical reasoning for planning and control: throw, balance, sweep, scoop, pack.","description":"Physical reasoning for robot planning and control, kept apart from perception and language: a simulated TidyBot++ mobile manipulator throws, balances, sweeps, scoops, stacks into cupboards, sorts, clears and packs. Each episode passes or fails on KinDER's own goal check (an object's centre inside a goal region, and the like), with no partial credit. We keep 5 of its 31 task families (the 4 hardest, scored, and one fixture), one variant each, and run each in two modes: unlimited (full simulator state and the goal check, unlimited resets, one replayable trajectory handed in) and limited (a robot behind a service: its base and wrist cameras, odometry and joint readings; one episode without resets).","edition":"5b2dbac (upstream), our selection of 4 tasks and a fixture (2026-10-04; 11 until then)","simulator":"MuJoCo 3.3 (KinDER's Dynamic3D environments)","links":{"paper":"https://arxiv.org/abs/2604.25788","repository":"https://github.com/Princeton-Robot-Planning-and-Learning/kindergarden","project_page":"https://prpl-group.com/kinder-site/"},"robots":[{"id":"tidybot-plus-plus","name":"TidyBot++","body":"mobile"}],"body":"mobile","bodies":{"mobile":5},"instances":5,"task_dirs":10,"modes":{"privileged":5,"standard":5},"oracle":{"full":1,"partial":0,"none":4},"protocols":["eai-standard/2.1"],"budgets_min":[30],"tags":{"planning-reasoning":3,"control-coordination":3,"mobile-whole-body-manipulation":3,"perception-understanding":1,"feedback-adaptation":1,"navigation-exploration":1},"on_site":true,"preview":[{"task":"sweep_into_drawer","title":"Dynamic3D · Sweep into drawer","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/sweep_into_drawer.jpg?v=b65dfc46","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/sweep_into_drawer.mp4?v=a4691bf6","motion":true,"aspect":1.778,"seconds":22},{"task":"constrained_cupboard","title":"Dynamic3D · Constrained cupboard","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/constrained_cupboard.jpg?v=a91f3451","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/constrained_cupboard.mp4?v=23417ce8","motion":true,"aspect":1.778,"seconds":22},{"task":"dynamo","title":"Dynamic3D · Dynamo","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/dynamo.jpg?v=df3ff8c4","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/dynamo.mp4?v=9ce4f946","motion":true,"aspect":1.778,"seconds":8},{"task":"scoop_pour_i01","title":"Scoop pour i01","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/scoop_pour_i01.jpg?v=c8d2dc4e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/scoop_pour_i01.mp4?v=76194a9e","motion":false,"aspect":1.778,"seconds":7},{"task":"sweep_simple_i01","title":"Sweep simple i01","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/sweep_simple_i01.jpg?v=47b5ebd7","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/sweep_simple_i01.mp4?v=a5122b0f","motion":false,"aspect":1.778,"seconds":7}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/_teaser.jpg?v=b65dfc46","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/_teaser.mp4?v=a4691bf6","cover":true,"aspect":1.778,"seconds":22},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":4,"success":0,"trials":4},"standard":{"done":4,"success":0,"trials":4}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":4,"success":0,"trials":4},"standard":{"done":4,"success":0,"trials":4}},"claude_code-fable_5_2-high":{"privileged":{"done":4,"success":2,"trials":4},"standard":{"done":4,"success":1,"trials":4}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":4,"success":1,"trials":4},"standard":{"done":4,"success":0,"trials":4}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":4,"success":0,"trials":4},"standard":{"done":4,"success":0,"trials":4}},"claude_code-opus_5_5-high":{"privileged":{"done":4,"success":1,"trials":4},"standard":{"done":4,"success":0,"trials":4}},"codex-gpt6_1_sol-high":{"privileged":{"done":4,"success":2,"trials":4},"standard":{"done":4,"success":1,"trials":4}},"codex-gpt6_2_astra-high":{"privileged":{"done":4,"success":3,"trials":4},"standard":{"done":4,"success":2,"trials":4}},"codex-gpt6_astra-high":{"privileged":{"done":4,"success":2,"trials":4},"standard":{"done":4,"success":0,"trials":4}},"codex-gpt6_luna-xhigh":{"privileged":{"done":4,"success":0,"trials":4},"standard":{"done":4,"success":0,"trials":4}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":4,"success":0,"trials":4},"standard":{"done":4,"success":0,"trials":4}}},"solved":{"privileged":4,"standard":3},"attempted":{"privileged":5,"standard":5},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":22.84,"est_n":8,"agent_s":12079,"agent_n":8},"antigravity_cli-gemini_4_argon-high":{"est_usd":17.15,"est_n":8,"agent_s":14429,"agent_n":8},"claude_code-fable_5_2-high":{"est_usd":38.23,"est_n":8,"agent_s":9060,"agent_n":8},"claude_code-glm_5_3_flash-high":{"est_usd":2.33,"est_n":8,"agent_s":11342,"agent_n":8},"claude_code-kimi_3_1_pro-high":{"est_usd":23.23,"est_n":8,"agent_s":12179,"agent_n":8},"claude_code-opus_5_5-high":{"est_usd":14.47,"est_n":8,"agent_s":6169,"agent_n":8},"codex-gpt6_1_sol-high":{"est_usd":7.63,"est_n":8,"agent_s":11394,"agent_n":8},"codex-gpt6_2_astra-high":{"est_usd":21.98,"est_n":8,"agent_s":7954,"agent_n":8},"codex-gpt6_astra-high":{"est_usd":65.29,"est_n":8,"agent_s":9823,"agent_n":8},"codex-gpt6_luna-xhigh":{"est_usd":1.84,"est_n":8,"agent_s":13607,"agent_n":8},"dsh-deepseek_v4_1_flash-high":{"est_usd":1.58,"est_n":8,"agent_s":11609,"agent_n":8}}},{"id":"robopaint","name":"RoboPaint","code":"RPT","origin":"ours","example":false,"blurb":"Tests fine tool control. A robot arm reproduces a given picture with a brush or pen, from Chinese calligraphy to oil painting.","description":"A Franka Panda holding a brush or a pointed pen copies a target picture onto a paper sheet, dipping in an ink stone or paint wells, with a practice sheet beside it: Chinese calligraphy in regular, running and cursive script, English copperplate lettering, and acrylic and oil paintings. Success needs the sheet to match the target closely (overlap, edges, stroke coverage and stray ink; for paintings, colour and structure) and the work to be done with strokes, the way a person writes or paints: filling the shape with thin back-and-forth lines, zigzags or spirals fails. Millimetre-wide strokes are hard to judge from camera images, and in standard mode the agent gets no score until the episode is over.","edition":"ours (2026-10-05)","simulator":"ManiSkill 3.0.1 / SAPIEN 3.0.3 + brush models (calligraphy, broad nib, libmypaint 1.6.1)","links":{"maniskill":"https://github.com/haosulab/ManiSkill","libmypaint":"https://github.com/mypaint/libmypaint","project_page":"https://robopaint.embodied-first-exam.ai"},"robots":[{"id":"franka-panda","name":"Franka Panda","body":"arm"}],"body":"arm","bodies":{"arm":43},"instances":43,"task_dirs":86,"modes":{"privileged":43,"standard":43},"oracle":{"full":0,"partial":43,"none":0},"protocols":["eai-standard/2.2"],"budgets_min":[60],"tags":{"perception-understanding":19,"planning-reasoning":19,"control-coordination":19,"manipulation":19},"on_site":true,"preview":[{"task":"oil_starrynight_i01","title":"Oil · Starrynight","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/scenes/oil_starrynight_i01.jpg?v=ab768d12","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/oil_starrynight_i01.mp4?v=2cb7e28e","motion":true,"aspect":1.778,"seconds":12},{"task":"oil_cafeterrace_i01","title":"Oil · Cafeterrace","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/scenes/oil_cafeterrace_i01.jpg?v=97dfd102","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/oil_cafeterrace_i01.mp4?v=485e7c0f","motion":true,"aspect":1.778,"seconds":12},{"task":"xingshu_huifenghechang","title":"Xingshu · Huifenghechang","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/scenes/xingshu_huifenghechang.jpg?v=f018251d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/xingshu_huifenghechang.mp4?v=3fc9d1d2","motion":true,"aspect":1.778,"seconds":12},{"task":"caoshu_beiquelindanshui","title":"Caoshu · Beiquelindanshui","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/scenes/caoshu_beiquelindanshui.jpg?v=891c4ec9","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/caoshu_beiquelindanshui.mp4?v=4e25ddeb","motion":true,"aspect":1.778,"seconds":12},{"task":"acrylic_redfuji","title":"Acrylic · Redfuji","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/scenes/acrylic_redfuji.jpg?v=b5e4615e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/acrylic_redfuji.mp4?v=6c3b988d","motion":true,"aspect":1.778,"seconds":12},{"task":"oil_irises_i01","title":"Oil · Irises","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/scenes/oil_irises_i01.jpg?v=ca555f6b","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/oil_irises_i01.mp4?v=0d93c892","motion":true,"aspect":1.778,"seconds":12},{"task":"xingshu_maolinxiuzhu","title":"Xingshu · Maolinxiuzhu","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/scenes/xingshu_maolinxiuzhu.jpg?v=5f041bb9","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/xingshu_maolinxiuzhu.mp4?v=916c788b","motion":true,"aspect":1.778,"seconds":12},{"task":"oil_monalisa_i01","title":"Oil · Monalisa","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/scenes/oil_monalisa_i01.jpg?v=3bf5e122","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/oil_monalisa_i01.mp4?v=58c9bf49","motion":true,"aspect":1.778,"seconds":12}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/_teaser.jpg?v=deabe02a","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/_teaser.mp4?v=2269df11","cover":true,"aspect":1.778,"seconds":26},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":22,"success":4,"trials":22},"standard":{"done":22,"success":0,"trials":22}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":22,"success":15,"trials":22},"standard":{"done":22,"success":2,"trials":22}},"claude_code-fable_5_2-high":{"privileged":{"done":22,"success":20,"trials":22},"standard":{"done":22,"success":2,"trials":22}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":22,"success":0,"trials":22},"standard":{"done":22,"success":0,"trials":22}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":22,"success":14,"trials":22},"standard":{"done":22,"success":0,"trials":22}},"claude_code-opus_5_5-high":{"privileged":{"done":22,"success":11,"trials":22},"standard":{"done":22,"success":2,"trials":22}},"codex-gpt6_1_sol-high":{"privileged":{"done":22,"success":20,"trials":22},"standard":{"done":22,"success":6,"trials":22}},"codex-gpt6_2_astra-high":{"privileged":{"done":22,"success":22,"trials":22},"standard":{"done":22,"success":21,"trials":22}},"codex-gpt6_astra-high":{"privileged":{"done":22,"success":22,"trials":22},"standard":{"done":22,"success":7,"trials":22}},"codex-gpt6_luna-xhigh":{"privileged":{"done":22,"success":0,"trials":22},"standard":{"done":22,"success":0,"trials":22}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":22,"success":1,"trials":22},"standard":{"done":22,"success":0,"trials":22}}},"solved":{"privileged":43,"standard":30},"attempted":{"privileged":43,"standard":43},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":257.11,"est_n":44,"agent_s":118133,"agent_n":44},"antigravity_cli-gemini_4_argon-high":{"est_usd":163.1,"est_n":44,"agent_s":134264,"agent_n":44},"claude_code-fable_5_2-high":{"est_usd":246.93,"est_n":44,"agent_s":75321,"agent_n":44},"claude_code-glm_5_3_flash-high":{"est_usd":26.18,"est_n":44,"agent_s":186095,"agent_n":44},"claude_code-kimi_3_1_pro-high":{"est_usd":212.49,"est_n":44,"agent_s":114891,"agent_n":44},"claude_code-opus_5_5-high":{"est_usd":128.2,"est_n":44,"agent_s":81610,"agent_n":44},"codex-gpt6_1_sol-high":{"est_usd":44.97,"est_n":44,"agent_s":91847,"agent_n":44},"codex-gpt6_2_astra-high":{"est_usd":161.59,"est_n":44,"agent_s":83114,"agent_n":44},"codex-gpt6_astra-high":{"est_usd":345.38,"est_n":44,"agent_s":86280,"agent_n":44},"codex-gpt6_luna-xhigh":{"est_usd":12.06,"est_n":44,"agent_s":95656,"agent_n":44},"dsh-deepseek_v4_1_flash-high":{"est_usd":18.88,"est_n":44,"agent_s":190446,"agent_n":44}}},{"id":"robotwin-2","name":"RoboTwin 2.0","code":"RTW","origin":"adapted","example":false,"blurb":"47 of its 50 bimanual tabletop tasks, for a dual-arm ALOHA-AgileX.","description":"47 of its 50 bimanual tabletop manipulation tasks on a dual-arm ALOHA-AgileX (four other embodiments supported), each defined as one Python class: a scene builder, a success predicate and a scripted expert that RoboTwin uses to generate its public 100k-episode dataset. Domain randomisation (clutter, textures, lighting, table height) is a switch we leave off. Success is the task's own `check_success()` at the end of the episode, all-or-nothing.","edition":"ICML 2026 paper (arXiv 2506.18088); code `main` @ 6dde571 (2026-09-14); assets HF revision 981c92a","simulator":"SAPIEN 3 (PhysX) + CuRobo","links":{"paper":"https://arxiv.org/abs/2506.18088","project_page":"https://robotwin-platform.github.io/","documentation":"https://robotwin-platform.github.io/doc/","tasks_doc":"https://robotwin-platform.github.io/doc/tasks/index.html","repository":"https://github.com/RoboTwin-Platform/RoboTwin","dataset":"https://huggingface.co/datasets/TianxingChen/RoboTwin2.0","leaderboard":"https://robotwin-platform.github.io/leaderboard"},"robots":[{"id":"aloha-agilex","name":"ALOHA AgileX","body":"bimanual"}],"body":"bimanual","bodies":{"bimanual":47},"instances":47,"task_dirs":94,"modes":{"privileged":47,"standard":47},"oracle":{"full":47,"partial":0,"none":0},"protocols":["eai-standard/2.1"],"budgets_min":[30],"tags":{"control-coordination":47,"manipulation":47,"planning-reasoning":21,"perception-understanding":18,"interaction-collaboration":2,"feedback-adaptation":1},"on_site":true,"preview":[{"task":"hanging_mug","title":"Hanging mug","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/hanging_mug.jpg?v=8f5f65f8","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/hanging_mug.mp4?v=cd6a6cc5","motion":true,"aspect":1.778,"seconds":29},{"task":"put_bottles_dustbin","title":"Put bottles dustbin","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/put_bottles_dustbin.jpg?v=5da402e1","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/put_bottles_dustbin.mp4?v=b9a70e4d","motion":true,"aspect":1.778,"seconds":31},{"task":"scan_object","title":"Scan object","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/scan_object.jpg?v=fdc6fecb","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/scan_object.mp4?v=df878b67","motion":true,"aspect":1.778,"seconds":32},{"task":"dump_bin_bigbin","title":"Dump bin big bin","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/dump_bin_bigbin.jpg?v=c0a4bff5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/dump_bin_bigbin.mp4?v=151f950a","motion":true,"aspect":1.778,"seconds":29},{"task":"place_can_basket","title":"Place can basket","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/place_can_basket.jpg?v=a0ee29af","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/place_can_basket.mp4?v=4301121d","motion":true,"aspect":1.778,"seconds":23},{"task":"stack_blocks_three","title":"Stack blocks three","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/stack_blocks_three.jpg?v=dfc97e3c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/stack_blocks_three.mp4?v=8306c6ab","motion":true,"aspect":1.778,"seconds":31},{"task":"blocks_ranking_rgb","title":"Blocks ranking rgb","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/blocks_ranking_rgb.jpg?v=3aec5083","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/blocks_ranking_rgb.mp4?v=620b824f","motion":true,"aspect":1.778,"seconds":22},{"task":"pick_diverse_bottles","title":"Pick diverse bottles","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/pick_diverse_bottles.jpg?v=9e8618b6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/pick_diverse_bottles.mp4?v=977fbdb1","motion":true,"aspect":1.778,"seconds":17}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/_teaser.jpg?v=8f5f65f8","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/_teaser.mp4?v=cd6a6cc5","cover":true,"aspect":1.778,"seconds":29},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":3,"success":1,"trials":3},"standard":{"done":3,"success":0,"trials":3}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":3,"success":3,"trials":3},"standard":{"done":3,"success":1,"trials":3}},"claude_code-fable_5_2-high":{"privileged":{"done":3,"success":3,"trials":3},"standard":{"done":3,"success":1,"trials":3}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":3,"success":0,"trials":3},"standard":{"done":3,"success":0,"trials":3}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":3,"success":1,"trials":3},"standard":{"done":3,"success":1,"trials":3}},"claude_code-opus_5_5-high":{"privileged":{"done":3,"success":3,"trials":3},"standard":{"done":3,"success":3,"trials":3}},"codex-gpt6_1_sol-high":{"privileged":{"done":3,"success":3,"trials":3},"standard":{"done":3,"success":2,"trials":3}},"codex-gpt6_2_astra-high":{"privileged":{"done":3,"success":3,"trials":3},"standard":{"done":3,"success":3,"trials":3}},"codex-gpt6_astra-high":{"privileged":{"done":3,"success":3,"trials":3},"standard":{"done":3,"success":1,"trials":3}},"codex-gpt6_luna-xhigh":{"privileged":{"done":3,"success":0,"trials":3},"standard":{"done":3,"success":0,"trials":3}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":3,"success":2,"trials":3},"standard":{"done":3,"success":0,"trials":3}}},"solved":{"privileged":45,"standard":31},"attempted":{"privileged":47,"standard":47},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":18.33,"est_n":6,"agent_s":9183,"agent_n":6},"antigravity_cli-gemini_4_argon-high":{"est_usd":10.49,"est_n":6,"agent_s":7428,"agent_n":6},"claude_code-fable_5_2-high":{"est_usd":14.41,"est_n":6,"agent_s":3769,"agent_n":6},"claude_code-glm_5_3_flash-high":{"est_usd":1.45,"est_n":6,"agent_s":8714,"agent_n":6},"claude_code-kimi_3_1_pro-high":{"est_usd":11.9,"est_n":6,"agent_s":9829,"agent_n":6},"claude_code-opus_5_5-high":{"est_usd":5.16,"est_n":6,"agent_s":2841,"agent_n":6},"codex-gpt6_1_sol-high":{"est_usd":5.28,"est_n":6,"agent_s":5934,"agent_n":6},"codex-gpt6_2_astra-high":{"est_usd":7.35,"est_n":6,"agent_s":2242,"agent_n":6},"codex-gpt6_astra-high":{"est_usd":35.76,"est_n":6,"agent_s":4957,"agent_n":6},"codex-gpt6_luna-xhigh":{"est_usd":1.28,"est_n":6,"agent_s":9695,"agent_n":6},"dsh-deepseek_v4_1_flash-high":{"est_usd":1.37,"est_n":6,"agent_s":8172,"agent_n":6}}},{"id":"mujoco-playground","name":"MuJoCo Playground","code":"MJP","origin":"adapted","example":false,"blurb":"Quadrupeds that walk, get up and balance, and manipulation for ALOHA 2 and a Franka Panda.","description":"Thirteen environments of Google DeepMind's MuJoCo Playground, run in plain MuJoCo on the CPU. Nine are quadruped locomotion: the Unitree Go1 tracks a joystick on flat and rough ground, gets up from a fall and balances in a handstand and a footstand; Boston Dynamics Spot tracks a joystick, with and without a commanded gait, and gets up; Google's Barkour vB tracks a joystick while it is pushed. Four are manipulation: ALOHA 2 hand-over and peg insertion, and a Franka Panda that opens a cabinet and places a cube in a given orientation. Playground trains with dense rewards, so each task has a success test of ours, built from its termination rules and reward targets.","edition":"MuJoCo Playground (arXiv 2502.08844); code `main` @ 4057c14","simulator":"MuJoCo 3.3.7 (plain MuJoCo on the CPU; upstream trains in MJX / MuJoCo Warp)","links":{"paper":"https://arxiv.org/abs/2502.08844","project_page":"https://playground.mujoco.org/","repository":"https://github.com/google-deepmind/mujoco_playground"},"robots":[{"id":"aloha-2","name":"ALOHA 2","body":"bimanual"},{"id":"boston-dynamics-spot","name":"Boston Dynamics Spot","body":"quadruped"},{"id":"franka-panda","name":"Franka Panda","body":"arm"},{"id":"google-barkour-vb","name":"Google Barkour vB","body":"quadruped"},{"id":"unitree-go1","name":"Unitree Go1","body":"quadruped"}],"body":"quadruped","bodies":{"bimanual":2,"arm":2,"quadruped":9},"instances":13,"task_dirs":27,"modes":{"privileged":13,"standard":13,"realtime":1},"oracle":{"full":13,"partial":0,"none":0},"protocols":["eai-standard/2.1","eai-standard/2.2","eai-standard/2.3"],"budgets_min":[30],"tags":{"planning-reasoning":2,"control-coordination":4,"manipulation":4,"interaction-collaboration":1},"on_site":true,"preview":[{"task":"go1_joystick_rough_terrain","title":"Go1 joystick rough terrain","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/go1_joystick_rough_terrain.jpg?v=3438b8d6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/go1_joystick_rough_terrain.mp4?v=50c16f04","motion":true,"aspect":1.778,"seconds":19},{"task":"aloha_single_peg_insertion","title":"Aloha Single Peg Insertion","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/aloha_single_peg_insertion.jpg?v=74a75082","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/aloha_single_peg_insertion.mp4?v=3b52460d","motion":true,"aspect":1.778,"seconds":38},{"task":"barkour_joystick","title":"Barkour joystick","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/barkour_joystick.jpg?v=9d89559c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/barkour_joystick.mp4?v=5cb6eadc","motion":true,"aspect":1.778,"seconds":13},{"task":"spot_joystick_gait_tracking","title":"Spot joystick gait tracking","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/spot_joystick_gait_tracking.jpg?v=ed93e414","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/spot_joystick_gait_tracking.mp4?v=c36bc1a0","motion":true,"aspect":1.778,"seconds":19},{"task":"aloha_hand_over","title":"Aloha Hand Over","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/aloha_hand_over.jpg?v=20f7a371","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/aloha_hand_over.mp4?v=8555db51","motion":true,"aspect":1.778,"seconds":26},{"task":"go1_joystick_flat_terrain","title":"Go1 joystick flat terrain","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/go1_joystick_flat_terrain.jpg?v=729ea116","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/go1_joystick_flat_terrain.mp4?v=246061ad","motion":true,"aspect":1.778,"seconds":19},{"task":"panda_open_cabinet","title":"Panda Open Cabinet","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/panda_open_cabinet.jpg?v=5ca79d71","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/panda_open_cabinet.mp4?v=444faeec","motion":true,"aspect":1.778,"seconds":14},{"task":"panda_pick_cube_orientation","title":"Panda Pick Cube Orientation","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/panda_pick_cube_orientation.jpg?v=fa4f25fb","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/panda_pick_cube_orientation.mp4?v=6961148b","motion":true,"aspect":1.778,"seconds":18}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/_teaser.jpg?v=3438b8d6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/_teaser.mp4?v=50c16f04","cover":true,"aspect":1.778,"seconds":19},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":7,"success":3,"trials":7},"standard":{"done":8,"success":0,"trials":8}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":7,"success":4,"trials":7},"standard":{"done":8,"success":2,"trials":8}},"claude_code-fable_5_2-high":{"privileged":{"done":7,"success":7,"trials":7},"standard":{"done":8,"success":3,"trials":8}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":7,"success":1,"trials":7},"standard":{"done":8,"success":0,"trials":8}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":7,"success":5,"trials":7},"standard":{"done":8,"success":0,"trials":8}},"claude_code-opus_5_5-high":{"privileged":{"done":7,"success":5,"trials":7},"standard":{"done":8,"success":0,"trials":8}},"codex-gpt6_1_sol-high":{"privileged":{"done":7,"success":7,"trials":7},"standard":{"done":8,"success":4,"trials":8}},"codex-gpt6_2_astra-high":{"privileged":{"done":7,"success":7,"trials":7},"standard":{"done":8,"success":8,"trials":8}},"codex-gpt6_astra-high":{"privileged":{"done":7,"success":7,"trials":7},"standard":{"done":8,"success":5,"trials":8}},"codex-gpt6_luna-xhigh":{"privileged":{"done":7,"success":0,"trials":7},"standard":{"done":8,"success":0,"trials":8}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":7,"success":3,"trials":7},"standard":{"done":8,"success":0,"trials":8}}},"solved":{"privileged":13,"standard":8},"attempted":{"privileged":13,"standard":13},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":36.11,"est_n":15,"agent_s":16738,"agent_n":15},"antigravity_cli-gemini_4_argon-high":{"est_usd":28.0,"est_n":15,"agent_s":24576,"agent_n":15},"claude_code-fable_5_2-high":{"est_usd":51.8,"est_n":15,"agent_s":7695,"agent_n":11},"claude_code-glm_5_3_flash-high":{"est_usd":3.66,"est_n":15,"agent_s":15971,"agent_n":15},"claude_code-kimi_3_1_pro-high":{"est_usd":31.69,"est_n":15,"agent_s":18682,"agent_n":15},"claude_code-opus_5_5-high":{"est_usd":19.45,"est_n":15,"agent_s":12065,"agent_n":15},"codex-gpt6_1_sol-high":{"est_usd":7.36,"est_n":15,"agent_s":15477,"agent_n":15},"codex-gpt6_2_astra-high":{"est_usd":18.4,"est_n":15,"agent_s":10223,"agent_n":15},"codex-gpt6_astra-high":{"est_usd":59.0,"est_n":15,"agent_s":11546,"agent_n":15},"codex-gpt6_luna-xhigh":{"est_usd":2.65,"est_n":15,"agent_s":20847,"agent_n":15},"dsh-deepseek_v4_1_flash-high":{"est_usd":3.39,"est_n":15,"agent_s":19579,"agent_n":15}}},{"id":"dextoolbench","name":"DexToolBench","code":"DTB","origin":"adapted","example":false,"blurb":"Dexterous tool use with a hammer, a marker, an eraser, a brush, a spatula or a screwdriver.","description":"21 of its 24 dexterous tool-use tasks from SimToolReal: 6 tool categories (hammer, marker, eraser, brush, spatula, screwdriver) x 2 real objects x 2 motions. Each task is a start pose and a sequence of goal poses for the tool, extracted from a human RGB-D demo; the score is task progress, the fraction of goals reached in order. Upstream runs it in Isaac Sim / Isaac Gym; we run SimToolReal's own MuJoCo sim2sim scene, rebuilt, with each category's table props.","edition":"SimToolReal (arXiv 2602.16863); code `main` @ 313d5ae (2026-08-13)","simulator":"MuJoCo 3.3.7 (our port; upstream runs Isaac Sim / Isaac Gym)","links":{"paper":"https://arxiv.org/abs/2602.16863","project_page":"https://simtoolreal.github.io/","repository":"https://github.com/tylerlum/simtoolreal"},"robots":[{"id":"kuka-iiwa14-sharpa-ha4","name":"KUKA iiwa14 + Sharpa HA4 hand","body":"hand"}],"body":"hand","bodies":{"hand":21},"instances":21,"task_dirs":42,"modes":{"privileged":21,"standard":21},"oracle":{"full":6,"partial":15,"none":0},"protocols":["eai-standard/2.1"],"budgets_min":[60],"tags":{"planning-reasoning":21,"control-coordination":21,"manipulation":21},"on_site":true,"preview":[{"task":"spatula_spoon_spatula_serve_plate","title":"Spoon spatula · serve plate","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/spatula_spoon_spatula_serve_plate.jpg?v=e5a68181","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/spatula_spoon_spatula_serve_plate.mp4?v=268b6861","motion":true,"aspect":1.778,"seconds":28},{"task":"hammer_claw_hammer_swing_side","title":"Claw hammer · swing side","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/hammer_claw_hammer_swing_side.jpg?v=972b161a","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/hammer_claw_hammer_swing_side.mp4?v=a27cc415","motion":true,"aspect":1.778,"seconds":24},{"task":"screwdriver_short_screwdriver_spin_vertical","title":"Short screwdriver · spin vertical","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/screwdriver_short_screwdriver_spin_vertical.jpg?v=e00f3e52","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/screwdriver_short_screwdriver_spin_vertical.mp4?v=b18c67bc","motion":true,"aspect":1.778,"seconds":28},{"task":"marker_staples_marker_draw_smile","title":"Staples marker · draw smile","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/marker_staples_marker_draw_smile.jpg?v=d389a3ce","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/marker_staples_marker_draw_smile.mp4?v=079f867c","motion":true,"aspect":1.778,"seconds":25},{"task":"eraser_handle_eraser_wipe_c","title":"Handle eraser · wipe c","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/eraser_handle_eraser_wipe_c.jpg?v=9136810c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/eraser_handle_eraser_wipe_c.mp4?v=e764103d","motion":true,"aspect":1.778,"seconds":26},{"task":"hammer_mallet_hammer_swing_down","title":"Mallet hammer · swing down","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/hammer_mallet_hammer_swing_down.jpg?v=9eed784f","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/hammer_mallet_hammer_swing_down.mp4?v=9a2590a6","motion":true,"aspect":1.778,"seconds":26},{"task":"hammer_mallet_hammer_swing_side","title":"Mallet hammer · swing side","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/hammer_mallet_hammer_swing_side.jpg?v=73fde17a","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/hammer_mallet_hammer_swing_side.mp4?v=36ec22d3","motion":true,"aspect":1.778,"seconds":23},{"task":"brush_red_brush_sweep_forward","title":"Red brush · sweep forward","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/brush_red_brush_sweep_forward.jpg?v=6430d06d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/brush_red_brush_sweep_forward.mp4?v=2512c945","motion":true,"aspect":1.778,"seconds":25}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/_teaser.jpg?v=e5a68181","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/_teaser.mp4?v=268b6861","cover":true,"aspect":1.778,"seconds":28},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":20,"success":2,"trials":20},"standard":{"done":20,"success":0,"trials":20}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":20,"success":1,"trials":20},"standard":{"done":20,"success":0,"trials":20}},"claude_code-fable_5_2-high":{"privileged":{"done":20,"success":10,"trials":20},"standard":{"done":20,"success":1,"trials":20}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":20,"success":0,"trials":20},"standard":{"done":20,"success":0,"trials":20}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":20,"success":0,"trials":20},"standard":{"done":20,"success":0,"trials":20}},"claude_code-opus_5_5-high":{"privileged":{"done":20,"success":0,"trials":20},"standard":{"done":20,"success":0,"trials":20}},"codex-gpt6_1_sol-high":{"privileged":{"done":20,"success":10,"trials":20},"standard":{"done":20,"success":1,"trials":20}},"codex-gpt6_2_astra-high":{"privileged":{"done":20,"success":17,"trials":20},"standard":{"done":20,"success":4,"trials":20}},"codex-gpt6_astra-high":{"privileged":{"done":20,"success":13,"trials":20},"standard":{"done":20,"success":2,"trials":20}},"codex-gpt6_luna-xhigh":{"privileged":{"done":20,"success":1,"trials":20},"standard":{"done":20,"success":0,"trials":20}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":20,"success":1,"trials":20},"standard":{"done":20,"success":0,"trials":20}}},"solved":{"privileged":19,"standard":7},"attempted":{"privileged":21,"standard":21},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":279.3,"est_n":40,"agent_s":130649,"agent_n":40},"antigravity_cli-gemini_4_argon-high":{"est_usd":185.3,"est_n":40,"agent_s":137535,"agent_n":40},"claude_code-fable_5_2-high":{"est_usd":472.7,"est_n":40,"agent_s":102961,"agent_n":40},"claude_code-glm_5_3_flash-high":{"est_usd":11.31,"est_n":40,"agent_s":74930,"agent_n":40},"claude_code-kimi_3_1_pro-high":{"est_usd":303.38,"est_n":40,"agent_s":138670,"agent_n":40},"claude_code-opus_5_5-high":{"est_usd":142.82,"est_n":40,"agent_s":56336,"agent_n":40},"codex-gpt6_1_sol-high":{"est_usd":87.88,"est_n":40,"agent_s":127336,"agent_n":40},"codex-gpt6_2_astra-high":{"est_usd":304.9,"est_n":40,"agent_s":104401,"agent_n":40},"codex-gpt6_astra-high":{"est_usd":820.39,"est_n":40,"agent_s":111803,"agent_n":40},"codex-gpt6_luna-xhigh":{"est_usd":18.53,"est_n":40,"agent_s":135368,"agent_n":40},"dsh-deepseek_v4_1_flash-high":{"est_usd":11.78,"est_n":40,"agent_s":88052,"agent_n":40}}},{"id":"dexjoco","name":"DexJoCo","code":"DXJ","origin":"adapted","example":false,"blurb":"A 16-DoF Allegro hand on a Franka arm, one hand or two.","description":"The 11 tasks of DexJoCo for a 16-joint Allegro hand on a Franka Panda arm, six with one arm and five with two side by side: spray a plant, click a mouse, drive a nail, lift a bucket, pinch with tongs, fold glasses into their case, move a three-disk Hanoi tower, cook a hot dog in a microwave, unlock a tablet, fit a peg into a socket and take a photograph. Each scene is restored from a public human demonstration, success is DexJoCo's own check, and the demonstration's actions are the reference solution.","simulator":"MuJoCo 3.4 (DexJoCo's gymnasium environments, CPU)","links":{"repository":"https://github.com/brave-eai/dexjoco","paper":"https://arxiv.org/abs/2605.16257","project_page":"https://dexjoco.github.io/"},"robots":[{"id":"franka-allegro","name":"Franka + Allegro hand","body":"hand"},{"id":"franka-allegro-bimanual","name":"Two Franka + Allegro hands","body":"hand"}],"body":"hand","bodies":{"hand":11},"instances":11,"task_dirs":22,"modes":{"privileged":11,"standard":11},"oracle":{"full":11,"partial":0,"none":0},"protocols":["eai-standard/2.2"],"budgets_min":[60],"on_site":false,"preview":[{"task":"water_plant","title":"Water plant","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/water_plant.jpg?v=2a53a472","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/water_plant.mp4?v=2bff4e6e","motion":true,"aspect":1.778,"seconds":8},{"task":"fold_glasses","title":"Fold glasses","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/fold_glasses.jpg?v=6da13f7d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/fold_glasses.mp4?v=a0e0e636","motion":true,"aspect":1.778,"seconds":14},{"task":"bimanual_photograph","title":"Bimanual photograph","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/bimanual_photograph.jpg?v=7e2389c4","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/bimanual_photograph.mp4?v=76a1d149","motion":true,"aspect":1.778,"seconds":7},{"task":"pick_bucket","title":"Pick bucket","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/pick_bucket.jpg?v=0ead3a87","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/pick_bucket.mp4?v=bbc981ce","motion":true,"aspect":1.778,"seconds":14},{"task":"hammer_nail","title":"Hammer nail","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/hammer_nail.jpg?v=85d55327","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/hammer_nail.mp4?v=71f2c43e","motion":true,"aspect":1.778,"seconds":6},{"task":"bimanual_assembly","title":"Bimanual assembly","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/bimanual_assembly.jpg?v=b0352e47","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/bimanual_assembly.mp4?v=334d8149","motion":true,"aspect":1.778,"seconds":14},{"task":"bimanual_hanoi","title":"Bimanual hanoi","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/bimanual_hanoi.jpg?v=3b413ab9","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/bimanual_hanoi.mp4?v=c4c34c88","motion":true,"aspect":1.778,"seconds":24},{"task":"bimanual_microwave_cook","title":"Bimanual microwave cook","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/bimanual_microwave_cook.jpg?v=5fe0ad2d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/bimanual_microwave_cook.mp4?v=8b03718c","motion":true,"aspect":1.778,"seconds":19}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/_teaser.jpg?v=f486ee23","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/_teaser.mp4?v=b71adfa3","cover":true,"aspect":1.778,"seconds":10},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":2,"success":1,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"claude_code-fable_5_2-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":2,"success":0,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":2,"success":1,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"claude_code-opus_5_5-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_1_sol-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"codex-gpt6_2_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"codex-gpt6_luna-xhigh":{"privileged":{"done":2,"success":0,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":2,"success":1,"trials":2},"standard":{"done":2,"success":0,"trials":2}}},"solved":{"privileged":11,"standard":7},"attempted":{"privileged":11,"standard":11},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":20.69,"est_n":4,"agent_s":10877,"agent_n":4},"antigravity_cli-gemini_4_argon-high":{"est_usd":11.07,"est_n":4,"agent_s":8087,"agent_n":4},"claude_code-fable_5_2-high":{"est_usd":31.36,"est_n":4,"agent_s":6164,"agent_n":4},"claude_code-glm_5_3_flash-high":{"est_usd":1.67,"est_n":4,"agent_s":8547,"agent_n":4},"claude_code-kimi_3_1_pro-high":{"est_usd":14.11,"est_n":4,"agent_s":11260,"agent_n":4},"claude_code-opus_5_5-high":{"est_usd":8.0,"est_n":4,"agent_s":5019,"agent_n":4},"codex-gpt6_1_sol-high":{"est_usd":6.5,"est_n":4,"agent_s":8128,"agent_n":4},"codex-gpt6_2_astra-high":{"est_usd":9.18,"est_n":4,"agent_s":3231,"agent_n":4},"codex-gpt6_astra-high":{"est_usd":40.73,"est_n":4,"agent_s":8723,"agent_n":4},"codex-gpt6_luna-xhigh":{"est_usd":1.03,"est_n":4,"agent_s":14397,"agent_n":4},"dsh-deepseek_v4_1_flash-high":{"est_usd":1.28,"est_n":4,"agent_s":9857,"agent_n":4}}},{"id":"myosuite","name":"MyoSuite","code":"MYO","origin":"adapted","example":false,"blurb":"Musculoskeletal hands, arms and legs, driven one muscle excitation at a time.","description":"Twenty-two MyoSuite tasks for musculoskeletal models of the human body: the MyoHand (39 muscles), the MyoArm, a table-tennis player with torso and arm, an arm next to a prosthetic arm, and the MyoLegs, one of them with a prosthetic leg. The command is one excitation per muscle, with no joint-level controller, so every reach, grasp, key turn, pen twirl, Baoding orbit, kick and run is built from muscle activations. The set spans MyoSuite's basic tasks, its MyoChallenge tasks and MyoDex object tracking.","simulator":"MyoSuite 2.12.2 (MuJoCo 3.6, CPU)","links":{"repository":"https://github.com/MyoHub/myosuite","paper":"https://arxiv.org/abs/2205.13600","project_page":"https://www.myosuite.org"},"robots":[{"id":"myoarm","name":"MyoArm","body":"musculoskeletal"},{"id":"myoarm-mpl","name":"MyoArm + prosthetic arm","body":"musculoskeletal"},{"id":"myoarm-torso","name":"MyoArm with torso","body":"musculoskeletal"},{"id":"myohand","name":"MyoHand","body":"musculoskeletal"},{"id":"myoleg-osl","name":"MyoLeg + prosthetic leg","body":"musculoskeletal"},{"id":"myolegs","name":"MyoLegs","body":"musculoskeletal"}],"body":"musculoskeletal","bodies":{"musculoskeletal":22},"instances":22,"task_dirs":45,"modes":{"privileged":22,"standard":22,"realtime":1},"oracle":{"full":3,"partial":0,"none":19},"protocols":["eai-standard/2.2","eai-standard/2.3"],"budgets_min":[60],"on_site":false,"preview":[{"task":"osl_run","title":"Osl run","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/osl_run.jpg?v=e0bcd8ed","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/osl_run.mp4?v=9408d5d4","motion":true,"aspect":1.778,"seconds":25},{"task":"baoding","title":"Baoding","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/baoding.jpg?v=10ecbc1e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/baoding.mp4?v=5fdf83bb","motion":true,"aspect":1.778,"seconds":8},{"task":"soccer","title":"Soccer","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/soccer.jpg?v=33c394c5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/soccer.mp4?v=6e36cd8c","motion":true,"aspect":1.778,"seconds":5},{"task":"table_tennis","title":"Table tennis","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/table_tennis.jpg?v=405fa27e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/table_tennis.mp4?v=c069906a","motion":true,"aspect":1.778,"seconds":6},{"task":"relocate","title":"Relocate","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/relocate.jpg?v=49ae0ee7","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/relocate.mp4?v=2e2bfa77","motion":true,"aspect":1.778,"seconds":7},{"task":"arm_reach","title":"Arm reach","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/arm_reach.jpg?v=036efd87","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/arm_reach.mp4?v=df0c7f85","motion":true,"aspect":1.778,"seconds":7},{"task":"chase_tag","title":"Chase tag","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/chase_tag.jpg?v=ff611687","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/chase_tag.mp4?v=a54c255d","motion":true,"aspect":1.778,"seconds":6},{"task":"dex_apple_lift","title":"Dex apple lift","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/dex_apple_lift.jpg?v=5e48ba0e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/dex_apple_lift.mp4?v=7497fd68","motion":true,"aspect":1.778,"seconds":6}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/_teaser.jpg?v=e0bcd8ed","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/_teaser.mp4?v=9408d5d4","cover":true,"aspect":1.778,"seconds":25},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":13,"success":2,"trials":13},"standard":{"done":13,"success":0,"trials":13}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":13,"success":0,"trials":13},"standard":{"done":13,"success":1,"trials":13}},"claude_code-fable_5_2-high":{"privileged":{"done":13,"success":6,"trials":13},"standard":{"done":13,"success":5,"trials":13}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":13,"success":0,"trials":13},"standard":{"done":13,"success":0,"trials":13}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":13,"success":3,"trials":13},"standard":{"done":13,"success":1,"trials":13}},"claude_code-opus_5_5-high":{"privileged":{"done":13,"success":5,"trials":13},"standard":{"done":13,"success":3,"trials":13}},"codex-gpt6_1_sol-high":{"privileged":{"done":13,"success":7,"trials":13},"standard":{"done":13,"success":3,"trials":13}},"codex-gpt6_2_astra-high":{"privileged":{"done":13,"success":8,"trials":13},"standard":{"done":13,"success":7,"trials":13}},"codex-gpt6_astra-high":{"privileged":{"done":13,"success":6,"trials":13},"standard":{"done":13,"success":3,"trials":13}},"codex-gpt6_luna-xhigh":{"privileged":{"done":13,"success":1,"trials":13},"standard":{"done":13,"success":0,"trials":13}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":13,"success":3,"trials":13},"standard":{"done":13,"success":0,"trials":13}}},"solved":{"privileged":17,"standard":9},"attempted":{"privileged":22,"standard":22},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":149.8,"est_n":26,"agent_s":63382,"agent_n":26},"antigravity_cli-gemini_4_argon-high":{"est_usd":91.22,"est_n":26,"agent_s":86573,"agent_n":26},"claude_code-fable_5_2-high":{"est_usd":215.19,"est_n":26,"agent_s":36477,"agent_n":15},"claude_code-glm_5_3_flash-high":{"est_usd":10.51,"est_n":26,"agent_s":59027,"agent_n":26},"claude_code-kimi_3_1_pro-high":{"est_usd":191.74,"est_n":26,"agent_s":89912,"agent_n":26},"claude_code-opus_5_5-high":{"est_usd":73.56,"est_n":26,"agent_s":50340,"agent_n":26},"codex-gpt6_1_sol-high":{"est_usd":64.5,"est_n":26,"agent_s":86824,"agent_n":26},"codex-gpt6_2_astra-high":{"est_usd":276.78,"est_n":26,"agent_s":68931,"agent_n":26},"codex-gpt6_astra-high":{"est_usd":495.43,"est_n":26,"agent_s":94103,"agent_n":26},"codex-gpt6_luna-xhigh":{"est_usd":6.9,"est_n":26,"agent_s":71882,"agent_n":26},"dsh-deepseek_v4_1_flash-high":{"est_usd":7.82,"est_n":26,"agent_s":59005,"agent_n":26}}},{"id":"roboarc","name":"RoboARC","code":"ARC","origin":"ours","example":false,"blurb":"Tests manipulation of tangled, deformable and multi-part objects. Robot arms take apart metal wire puzzles, tie and thread ropes, fold cloth and assemble IKEA furniture.","description":"Physical puzzles for the two arms of an ALOHA 2 or a Franka arm with a Robotiq gripper: separate interlocked claws, take a ring off a horseshoe or a slotted post, free a hedgehog from its cage, fit two cast-puzzle pieces together, turn a key through a lock, guide a loop along a buzz wire without touching it, untie a knot, thread a rope or a fiber bundle through rings, flatten or fold a blanket. With them, IKEA furniture builds for two arms that hold the parts by finger friction alone: screw the legs into a LACK table, fit the dowels and panels of an EKET cabinet, close the snaps of a VESKEN shelf, drive bolts with a hex key. The simulator judges the end state (the piece in its goal region, the rope free of crossings, every part seated and at rest after the robot lets go), and a success won by pushing parts deep into each other is held for review rather than credited. The hard part is working out the shape of the puzzle before touching it, then handling ropes, cloth and tight fits with a gripper.","simulator":"MuJoCo 3.14 (CPU float64; the cloth on a deterministic GPU solver); videos rendered with Blender Cycles","links":{"project_page":"https://roboarc.embodied-first-exam.ai"},"robots":[{"id":"aloha-2","name":"ALOHA 2","body":"bimanual"},{"id":"franka-panda-bimanual","name":"Two Franka Panda arms","body":"bimanual"},{"id":"franka-robotiq-2f85","name":"Franka + Robotiq 2F-85","body":"arm"}],"body":"bimanual","bodies":{"bimanual":73,"arm":33},"instances":106,"task_dirs":212,"modes":{"privileged":106,"standard":106},"oracle":{"full":104,"partial":0,"none":2},"protocols":["eai-standard/2.4"],"budgets_min":[180],"on_site":false,"preview":[{"task":"interlocked_claws_05","title":"Interlocked claws 05","cover":true,"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/interlocked_claws_05.jpg?v=27ce3fd0","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/interlocked_claws_05.mp4?v=da9a54fc","motion":true,"aspect":1.778,"seconds":42},{"task":"interlocked_claws_12","title":"Interlocked claws 12","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/interlocked_claws_12.jpg?v=aa77c868","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/interlocked_claws_12.mp4?v=ad315c21","motion":true,"aspect":1.778,"seconds":42},{"task":"interlocked_claws_10","title":"Interlocked claws 10","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/interlocked_claws_10.jpg?v=7eb72003","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/interlocked_claws_10.mp4?v=b2fbff69","motion":true,"aspect":1.778,"seconds":22},{"task":"cast_keyhole_06","title":"Cast keyhole 06","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/cast_keyhole_06.jpg?v=361bebe6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/cast_keyhole_06.mp4?v=66adaea3","motion":true,"aspect":1.778,"seconds":28},{"task":"rope_threading_08","title":"Rope threading 08","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/rope_threading_08.jpg?v=3694df15","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/rope_threading_08.mp4?v=a32bb4a1","motion":true,"aspect":1.778,"seconds":33},{"task":"knot_untying_05","title":"Knot untying 05","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/knot_untying_05.jpg?v=72d00984","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/knot_untying_05.mp4?v=8abcf009","motion":true,"aspect":1.778,"seconds":17},{"task":"hedgehog_in_cage_05","title":"Hedgehog in cage 05","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/hedgehog_in_cage_05.jpg?v=b4508cd1","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/hedgehog_in_cage_05.mp4?v=c00ffe6b","motion":true,"aspect":1.778,"seconds":22},{"task":"slotted_post_ring_05","title":"Slotted post ring 05","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/slotted_post_ring_05.jpg?v=bc2f8765","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/slotted_post_ring_05.mp4?v=f4e27760","motion":true,"aspect":1.778,"seconds":32},{"task":"buzz_wire_3d_10","title":"Buzz wire 3d 10","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/buzz_wire_3d_10.jpg?v=4c559f53","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/buzz_wire_3d_10.mp4?v=a40d3d5e","motion":true,"aspect":1.778,"seconds":42},{"task":"ikea_vesken","title":"IKEA Vesken","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_vesken.jpg?v=f09c5a47","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_vesken.mp4?v=76bb3e23","motion":true,"aspect":1.778,"seconds":42},{"task":"ikea_gladom","title":"IKEA Gladom","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_gladom.jpg?v=40ab4c20","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_gladom.mp4?v=e7f44f41","motion":true,"aspect":1.778,"seconds":22},{"task":"ikea_eket","title":"IKEA Eket","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_eket.jpg?v=a41b4f69","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_eket.mp4?v=0683c826","motion":true,"aspect":1.778,"seconds":24},{"task":"ikea_ikea365","title":"IKEA 365+","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_ikea365.jpg?v=0126d1d5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_ikea365.mp4?v=4334a742","motion":true,"aspect":1.778,"seconds":30},{"task":"ikea_variera","title":"IKEA Variera","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_variera.jpg?v=1d8db5d6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_variera.mp4?v=912752e3","motion":true,"aspect":1.778,"seconds":22},{"task":"ikea_stegesund","title":"IKEA Stegesund","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_stegesund.jpg?v=b348f6cd","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_stegesund.mp4?v=7c58a1fa","motion":true,"aspect":1.778,"seconds":32},{"task":"ikea_svallet","title":"IKEA Svallet","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_svallet.jpg?v=9d89b227","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_svallet.mp4?v=15ed594f","motion":true,"aspect":1.778,"seconds":24}],"real_robot":[{"video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/real_buzzwire.mp4?v=2447dcbc","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/real_buzzwire.jpg?v=ccb66bbc","aspect":3.019,"caption":"The buzz-wire task on a real robot arm, run by GPT-6 Astra in Codex (left) and Claude Opus 5.5 in Claude Code (right): guide the loop along the wire without touching it. The inset shows the buzzer's bulb, which lights when the loop touches the wire; the label under each name gives the playback speed of its thinking and acting."}],"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":58,"success":5,"trials":58},"standard":{"done":58,"success":2,"trials":58}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":58,"success":27,"trials":58},"standard":{"done":58,"success":11,"trials":58}},"claude_code-fable_5_2-high":{"privileged":{"done":58,"success":53,"trials":58},"standard":{"done":58,"success":33,"trials":58}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":58,"success":1,"trials":58},"standard":{"done":58,"success":0,"trials":58}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":58,"success":13,"trials":58},"standard":{"done":58,"success":1,"trials":58}},"claude_code-opus_5_5-high":{"privileged":{"done":58,"success":43,"trials":58},"standard":{"done":58,"success":15,"trials":58}},"codex-gpt6_1_sol-high":{"privileged":{"done":58,"success":40,"trials":58},"standard":{"done":58,"success":14,"trials":58}},"codex-gpt6_2_astra-high":{"privileged":{"done":58,"success":58,"trials":58},"standard":{"done":58,"success":32,"trials":58}},"codex-gpt6_astra-high":{"privileged":{"done":58,"success":47,"trials":58},"standard":{"done":58,"success":15,"trials":58}},"codex-gpt6_luna-xhigh":{"privileged":{"done":58,"success":3,"trials":58},"standard":{"done":58,"success":0,"trials":58}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":58,"success":5,"trials":58},"standard":{"done":58,"success":2,"trials":58}}},"solved":{"privileged":106,"standard":76},"attempted":{"privileged":106,"standard":106},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":1540.11,"est_n":116,"agent_s":1098926,"agent_n":116},"antigravity_cli-gemini_4_argon-high":{"est_usd":1013.23,"est_n":116,"agent_s":889101,"agent_n":116},"claude_code-fable_5_2-high":{"est_usd":2483.99,"est_n":115,"agent_s":690170,"agent_n":105},"claude_code-glm_5_3_flash-high":{"est_usd":109.51,"est_n":116,"agent_s":995445,"agent_n":116},"claude_code-kimi_3_1_pro-high":{"est_usd":1760.57,"est_n":116,"agent_s":1165924,"agent_n":116},"claude_code-opus_5_5-high":{"est_usd":1231.44,"est_n":116,"agent_s":637342,"agent_n":116},"codex-gpt6_1_sol-high":{"est_usd":852.2,"est_n":116,"agent_s":924627,"agent_n":116},"codex-gpt6_2_astra-high":{"est_usd":1279.22,"est_n":116,"agent_s":555431,"agent_n":116},"codex-gpt6_astra-high":{"est_usd":4872.81,"est_n":116,"agent_s":854127,"agent_n":116},"codex-gpt6_luna-xhigh":{"est_usd":124.58,"est_n":116,"agent_s":1147555,"agent_n":116},"dsh-deepseek_v4_1_flash-high":{"est_usd":82.57,"est_n":116,"agent_s":875909,"agent_n":116}}},{"id":"robodojo","name":"RoboDojo","code":"RDJ","origin":"adapted","example":false,"blurb":"Bimanual manipulation over five capabilities: generalization, memory, precision, long horizons, open tasks.","description":"The simulation part of RoboDojo's benchmark for generalist manipulation policies: two ARX X5 arms with parallel grippers at a table. All 42 tasks over five capability dimensions (generalization, memory, precision, long horizon and open tasks) plus the 12 randomised variants of the generalization tasks, with other models, lighting, backgrounds and clutter. Each task is one frozen evaluation layout, in a privileged and a standard mode. A privileged answer is replayed up to three times, each in a fresh process, and counts as solved if any replay succeeds: the physics of six tasks (cloth, liquids, toast) does not replay bit for bit.","simulator":"Isaac Lab 2.3 / Isaac Sim 5.1","links":{"repository":"https://github.com/RoboDojo-Benchmark/RoboDojo","paper":"https://arxiv.org/abs/2607.04434","project_page":"https://robodojo-benchmark.com/"},"robots":[{"id":"arx-x5-bimanual","name":"Two ARX X5 arms","body":"bimanual"}],"body":"bimanual","bodies":{"bimanual":54},"instances":54,"task_dirs":108,"modes":{"privileged":54,"standard":54},"oracle":{"full":0,"partial":0,"none":54},"protocols":["eai-standard/2.2"],"budgets_min":[60],"on_site":false,"preview":[{"task":"hang_mugs_random","title":"Hang mugs random","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/hang_mugs_random.jpg?v=c992d260","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/hang_mugs_random.mp4?v=a0fda6fb","motion":true,"aspect":1.778,"seconds":25},{"task":"stack_blocks_by_language","title":"Stack blocks by language","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/stack_blocks_by_language.jpg?v=af1bb18f","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/stack_blocks_by_language.mp4?v=388e6529","motion":true,"aspect":1.778,"seconds":12},{"task":"arrange_largest_number","title":"Arrange largest number","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/arrange_largest_number.jpg?v=69b676c1","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/arrange_largest_number.mp4?v=87e86eee","motion":true,"aspect":1.778,"seconds":19},{"task":"match_and_pick_from_conveyor","title":"Match and pick from conveyor","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/match_and_pick_from_conveyor.jpg?v=85c4a487","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/match_and_pick_from_conveyor.mp4?v=870798e9","motion":true,"aspect":1.778,"seconds":7},{"task":"pour_by_language","title":"Pour by language","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/pour_by_language.jpg?v=89d37569","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/pour_by_language.mp4?v=858dd74c","motion":true,"aspect":1.778,"seconds":28},{"task":"make_toast_random","title":"Make toast random","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/make_toast_random.jpg?v=bdbc2a6d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/make_toast_random.mp4?v=e4d7b1ac","motion":true,"aspect":1.778,"seconds":29},{"task":"align_blocks","title":"Align blocks","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/align_blocks.jpg?v=2d23f7b0","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/align_blocks.mp4?v=5d3842f4","motion":true,"aspect":1.778,"seconds":14},{"task":"arrange_largest_number_random","title":"Arrange largest number random","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/arrange_largest_number_random.jpg?v=f22daa6e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/arrange_largest_number_random.mp4?v=d1dd1b3a","motion":true,"aspect":1.778,"seconds":21}],"prompts":{"title":"Three prompt versions.","text":"RoboDojo's 37 core tasks in the Standard Setting, run by GPT-6 Astra and Claude Fable 5.2 with three prompts; scenes, step limits and success checkers are the same.","models":["GPT-6 Astra","Claude Fable 5.2"],"items":[{"name":"RoboDojo VLA prompt","text":"RoboDojo's one-sentence task description.","solved":[[6,37],[21,37]]},{"name":"RoboDojo agent prompt","text":"the same sentence plus the task recipe and the return-to-start note from RoboDojo's own agent evaluation.","solved":[[17,37],[24,37]]},{"name":"Our prompt","text":"the task stated together with every condition RoboDojo's success checker tests, including that both arms end near their start pose.","solved":[[21,37],[28,37]]}],"after":"The leaderboard uses our prompt.","example":{"title":"Example: Fill Egg Holder","prompts":[{"name":"RoboDojo VLA prompt","blocks":["**Task:** Place the four eggs from the basket into the egg holder, then close the lid."]},{"name":"RoboDojo agent prompt","blocks":["**Task:** Place the four eggs from the basket into the egg holder, then close the lid.","**Description.** There is a woven basket containing four eggs and an egg holder. The robot needs to pick up the eggs one by one, place all four into the egg holder, and then close the lid.","**Scoring.**","- 0: No egg is placed into the egg holder.","- 10: Any one egg is inside the egg holder and close to the holder bottom, with the gripper open.","- 25: Any two eggs are placed correctly into the egg holder.","- 40: Any three eggs are placed correctly into the egg holder.","- 90: All four eggs are placed in the holder, but the lid is not fully closed yet.","- 100: All four eggs are in the holder, the holder lid is correctly and fully closed, and the robot returns to origin.","Most of these tasks also score the arms themselves: the work does not score until both arms are back within 0.15 m and 20 degrees of the pose they began the episode in, with the objects left as the task asked. Getting home is the last step of the task rather than tidying up afterwards, so keep enough env steps in hand to make it."]},{"name":"Our prompt","blocks":["**Task:** Move the four eggs from the egg basket into the egg holder (an articulated tray with a lid), close the lid, and return the arm home.","**Success.** All of these must hold at the same action boundary, for each of the four eggs (any egg in any cup):","- **In the holder:** the egg's centre … seen from above, inside the holder's outline … and higher than that outline's lowest point.","- **Seated in a cup:** the egg's centre strictly between 0.01 and 0.04 m higher than the carton's floor point … An egg resting in a cup passes; an egg on top of the closed lid, on the rim or held above the carton fails.","- **Lid closed:** the lid's hinge more than 90 % of the way along its range toward closed …","- **Both arms back:** each arm's end-effector within 0.15 m of its start position on each of x, y and z separately, and within 20 deg of its start orientation. The grippers need not be open."]}],"after":"Full prompts also include the robot client, rules and time limit, the same in all three."}},"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/_teaser.jpg?v=c992d260","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/_teaser.mp4?v=a0fda6fb","cover":true,"aspect":1.778,"seconds":25},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":37,"success":9,"trials":37},"standard":{"done":37,"success":3,"trials":37}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":37,"success":24,"trials":37},"standard":{"done":37,"success":21,"trials":37}},"claude_code-fable_5_2-high":{"privileged":{"done":37,"success":36,"trials":37},"standard":{"done":37,"success":28,"trials":37}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":37,"success":6,"trials":37},"standard":{"done":37,"success":1,"trials":37}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":37,"success":19,"trials":37},"standard":{"done":37,"success":10,"trials":37}},"claude_code-opus_5_5-high":{"privileged":{"done":37,"success":27,"trials":37},"standard":{"done":37,"success":18,"trials":37}},"codex-gpt6_1_sol-high":{"privileged":{"done":37,"success":29,"trials":37},"standard":{"done":37,"success":16,"trials":37}},"codex-gpt6_2_astra-high":{"privileged":{"done":37,"success":37,"trials":37},"standard":{"done":37,"success":22,"trials":37}},"codex-gpt6_astra-high":{"privileged":{"done":37,"success":35,"trials":37},"standard":{"done":37,"success":21,"trials":37}},"codex-gpt6_luna-xhigh":{"privileged":{"done":37,"success":0,"trials":37},"standard":{"done":37,"success":0,"trials":37}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":37,"success":7,"trials":37},"standard":{"done":37,"success":3,"trials":37}}},"solved":{"privileged":54,"standard":48},"attempted":{"privileged":54,"standard":54},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":190.74,"est_n":69,"agent_s":158059,"agent_n":74},"antigravity_cli-gemini_4_argon-high":{"est_usd":191.36,"est_n":69,"agent_s":181933,"agent_n":74},"claude_code-fable_5_2-high":{"est_usd":469.63,"est_n":69,"agent_s":126414,"agent_n":74},"claude_code-glm_5_3_flash-high":{"est_usd":29.02,"est_n":69,"agent_s":189684,"agent_n":74},"claude_code-kimi_3_1_pro-high":{"est_usd":289.03,"est_n":69,"agent_s":177433,"agent_n":74},"claude_code-opus_5_5-high":{"est_usd":114.34,"est_n":47,"agent_s":80707,"agent_n":52},"codex-gpt6_1_sol-high":{"est_usd":110.38,"est_n":69,"agent_s":163820,"agent_n":74},"codex-gpt6_2_astra-high":{"est_usd":206.57,"est_n":74,"agent_s":97265,"agent_n":74},"codex-gpt6_astra-high":{"est_usd":703.12,"est_n":69,"agent_s":133096,"agent_n":74},"codex-gpt6_luna-xhigh":{"est_usd":18.46,"est_n":74,"agent_s":229670,"agent_n":74},"dsh-deepseek_v4_1_flash-high":{"est_usd":23.59,"est_n":69,"agent_s":186310,"agent_n":74}}},{"id":"pandadyn","name":"RoboDynamicExam","code":"PDS","origin":"ours","example":false,"blurb":"Tests handling of moving objects. One or two robot arms catch, strike, carry and hand over objects in motion: tossed balls, sliding pucks, swinging loads.","description":"Multi-stage dynamic manipulation for one or two Franka Panda arms, most of them holding a rigid tool such as a plate, a cup, a paddle or a scoop. In every task something moves by itself (a tossed ball, a sliding puck, a conveyor, a pendulum, a spring gate, a moving cart), and the robot has to catch, strike, carry, flip, hand over or release at the right moment, several times in an episode of under a minute of simulated time. Success is every phase of the task in order with no failure on the way, checked from the simulator state, so a ball dropped in the third phase fails the whole task. Most tasks also run in real time, where the world keeps moving while the agent thinks.","simulator":"ManiSkill 3 / SAPIEN (CPU PhysX at 1000 Hz)","robots":[{"id":"franka-panda","name":"Franka Panda","body":"arm"},{"id":"franka-panda-bimanual","name":"Two Franka Panda arms","body":"bimanual"}],"body":"bimanual","bodies":{"bimanual":13,"arm":12},"instances":25,"task_dirs":69,"modes":{"privileged":25,"standard":25,"realtime":19},"oracle":{"full":25,"partial":0,"none":0},"protocols":["eai-standard/2.1","eai-standard/2.3"],"budgets_min":[60],"on_site":false,"preview":[{"task":"d01_reciprocal_cup_relay","title":"D01 reciprocal cup relay","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/d01_reciprocal_cup_relay.jpg?v=2af79c2f","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/d01_reciprocal_cup_relay.mp4?v=36755819","motion":true,"aspect":1.778,"seconds":16},{"task":"d09_lever_launch_catch_reload","title":"D09 lever launch catch reload","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/d09_lever_launch_catch_reload.jpg?v=4dbaea9f","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/d09_lever_launch_catch_reload.mp4?v=b6e434bd","motion":true,"aspect":1.778,"seconds":17},{"task":"d02_strike_catch_merge","title":"D02 strike catch merge","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/d02_strike_catch_merge.jpg?v=2cdffff3","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/d02_strike_catch_merge.mp4?v=bb4b8d32","motion":true,"aspect":1.778,"seconds":19},{"task":"d05_moving_ramp_relay","title":"D05 moving ramp relay","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/d05_moving_ramp_relay.jpg?v=fa2bddaa","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/d05_moving_ramp_relay.mp4?v=95e7c0fd","motion":true,"aspect":1.778,"seconds":25},{"task":"d10_moving_grasp_handover","title":"D10 moving grasp handover","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/d10_moving_grasp_handover.jpg?v=7df83713","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/d10_moving_grasp_handover.mp4?v=0d09289b","motion":true,"aspect":1.778,"seconds":14},{"task":"s09_toss_recatch_delivery","title":"S09 toss recatch delivery","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/s09_toss_recatch_delivery.jpg?v=7633d270","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/s09_toss_recatch_delivery.mp4?v=17ba0dba","motion":true,"aspect":1.778,"seconds":13},{"task":"s12_moving_bridge_puck_relay","title":"S12 moving bridge puck relay","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/s12_moving_bridge_puck_relay.jpg?v=3f256af4","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/s12_moving_bridge_puck_relay.mp4?v=0ac6f103","motion":true,"aspect":1.778,"seconds":32},{"task":"d03_cooperative_puck_course","title":"D03 cooperative puck course","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/d03_cooperative_puck_course.jpg?v=c63ced34","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/d03_cooperative_puck_course.mp4?v=d513ef7d","motion":true,"aspect":1.778,"seconds":24}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/_teaser.jpg?v=2af79c2f","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/_teaser.mp4?v=36755819","cover":true,"aspect":1.778,"seconds":16},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":10,"success":2,"trials":10},"standard":{"done":18,"success":0,"trials":18}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":10,"success":5,"trials":10},"standard":{"done":18,"success":2,"trials":18}},"claude_code-fable_5_2-high":{"privileged":{"done":10,"success":10,"trials":10},"standard":{"done":18,"success":10,"trials":18}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":10,"success":1,"trials":10},"standard":{"done":18,"success":0,"trials":18}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":10,"success":7,"trials":10},"standard":{"done":18,"success":0,"trials":18}},"claude_code-opus_5_5-high":{"privileged":{"done":10,"success":10,"trials":10},"standard":{"done":18,"success":2,"trials":18}},"codex-gpt6_1_sol-high":{"privileged":{"done":10,"success":10,"trials":10},"standard":{"done":18,"success":5,"trials":18}},"codex-gpt6_2_astra-high":{"privileged":{"done":10,"success":10,"trials":10},"standard":{"done":18,"success":11,"trials":18}},"codex-gpt6_astra-high":{"privileged":{"done":10,"success":10,"trials":10},"standard":{"done":18,"success":5,"trials":18}},"codex-gpt6_luna-xhigh":{"privileged":{"done":10,"success":0,"trials":10},"standard":{"done":18,"success":0,"trials":18}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":10,"success":2,"trials":10},"standard":{"done":18,"success":0,"trials":18}}},"solved":{"privileged":25,"standard":8},"attempted":{"privileged":25,"standard":25},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":92.5,"est_n":28,"agent_s":47064,"agent_n":28},"antigravity_cli-gemini_4_argon-high":{"est_usd":100.49,"est_n":28,"agent_s":71985,"agent_n":28},"claude_code-fable_5_2-high":{"est_usd":224.62,"est_n":28,"agent_s":47417,"agent_n":28},"claude_code-glm_5_3_flash-high":{"est_usd":12.69,"est_n":28,"agent_s":60871,"agent_n":28},"claude_code-kimi_3_1_pro-high":{"est_usd":93.81,"est_n":28,"agent_s":60857,"agent_n":28},"claude_code-opus_5_5-high":{"est_usd":79.32,"est_n":28,"agent_s":36476,"agent_n":28},"codex-gpt6_1_sol-high":{"est_usd":32.15,"est_n":28,"agent_s":51751,"agent_n":28},"codex-gpt6_2_astra-high":{"est_usd":47.07,"est_n":28,"agent_s":17143,"agent_n":28},"codex-gpt6_astra-high":{"est_usd":187.79,"est_n":28,"agent_s":32124,"agent_n":28},"codex-gpt6_luna-xhigh":{"est_usd":6.88,"est_n":28,"agent_s":56880,"agent_n":28},"dsh-deepseek_v4_1_flash-high":{"est_usd":10.22,"est_n":28,"agent_s":60601,"agent_n":28}}},{"id":"bigym2","name":"BiGym 2.0","code":"BG2","origin":"adapted","example":false,"blurb":"Household loco-manipulation on a walking Unitree G1.","description":"Household tasks such as a dishwasher, cupboards and drawers for a Unitree G1 with two grippers, kept balanced and walking by NVIDIA's GR00T-WBC policy. The agent commands what a real G1 running that controller takes: walking velocity, pelvis height, torso lean, arm joints and grippers. 13 families, each also in realtime mode, where the world keeps running while the agent thinks.","simulator":"MuJoCo 3.8 + NVIDIA GR00T-WBC","links":{"repository":"https://github.com/swirl-uk/BiGym2","paper":"https://arxiv.org/abs/2610.07594","project_page":"https://bigym2.github.io/"},"robots":[{"id":"unitree-g1-dex1","name":"Unitree G1 with grippers","body":"humanoid"}],"body":"humanoid","bodies":{"humanoid":13},"instances":13,"task_dirs":39,"modes":{"privileged":13,"standard":13,"realtime":13},"oracle":{"full":0,"partial":0,"none":13},"protocols":["eai-standard/2.1","eai-standard/2.3"],"budgets_min":[60],"on_site":false,"preview":[{"task":"dishwasher_load_plates","title":"Dishwasher load plates","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/dishwasher_load_plates.jpg?v=ebac1404","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/dishwasher_load_plates.mp4?v=a1ad93f6","motion":true,"aspect":1.778,"seconds":22},{"task":"put_cups","title":"Put cups","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/put_cups.jpg?v=8f52a614","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/put_cups.mp4?v=b7ad7447","motion":true,"aspect":1.778,"seconds":22},{"task":"saucepan_to_hob","title":"Saucepan to hob","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/saucepan_to_hob.jpg?v=c60515fe","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/saucepan_to_hob.mp4?v=c6074eb2","motion":true,"aspect":1.778,"seconds":22},{"task":"pick_box","title":"Pick box","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/pick_box.jpg?v=0b481451","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/pick_box.mp4?v=2c610453","motion":true,"aspect":1.778,"seconds":22},{"task":"sandwich_remove","title":"Sandwich remove","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/sandwich_remove.jpg?v=075ba132","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/sandwich_remove.mp4?v=c527fea7","motion":true,"aspect":1.778,"seconds":22},{"task":"dishwasher_close","title":"Dishwasher close","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/dishwasher_close.jpg?v=2a678deb","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/dishwasher_close.mp4?v=25e2c941","motion":true,"aspect":1.778,"seconds":22},{"task":"drawer_top_open","title":"Drawer top open","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/drawer_top_open.jpg?v=bfb2a8bb","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/drawer_top_open.mp4?v=023e6ae5","motion":true,"aspect":1.778,"seconds":9},{"task":"flip_cup","title":"Flip cup","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/flip_cup.jpg?v=5443b8e8","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/flip_cup.mp4?v=aef45b8c","motion":true,"aspect":1.778,"seconds":7}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/_teaser.jpg?v=ebac1404","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/bigym2/_teaser.mp4?v=a1ad93f6","cover":true,"aspect":1.778,"seconds":22},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":10,"success":6,"trials":10},"standard":{"done":20,"success":0,"trials":20}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":10,"success":7,"trials":10},"standard":{"done":20,"success":2,"trials":20}},"claude_code-fable_5_2-high":{"privileged":{"done":10,"success":10,"trials":10},"standard":{"done":20,"success":7,"trials":20}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":10,"success":0,"trials":10},"standard":{"done":20,"success":0,"trials":20}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":10,"success":8,"trials":10},"standard":{"done":20,"success":0,"trials":20}},"claude_code-opus_5_5-high":{"privileged":{"done":10,"success":5,"trials":10},"standard":{"done":20,"success":3,"trials":20}},"codex-gpt6_1_sol-high":{"privileged":{"done":10,"success":9,"trials":10},"standard":{"done":20,"success":0,"trials":20}},"codex-gpt6_2_astra-high":{"privileged":{"done":10,"success":10,"trials":10},"standard":{"done":20,"success":8,"trials":20}},"codex-gpt6_astra-high":{"privileged":{"done":10,"success":10,"trials":10},"standard":{"done":20,"success":5,"trials":20}},"codex-gpt6_luna-xhigh":{"privileged":{"done":10,"success":1,"trials":10},"standard":{"done":20,"success":0,"trials":20}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":10,"success":0,"trials":10},"standard":{"done":20,"success":0,"trials":20}}},"solved":{"privileged":13,"standard":9},"attempted":{"privileged":13,"standard":13},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":108.65,"est_n":30,"agent_s":50450,"agent_n":30},"antigravity_cli-gemini_4_argon-high":{"est_usd":112.03,"est_n":30,"agent_s":80659,"agent_n":30},"claude_code-fable_5_2-high":{"est_usd":252.79,"est_n":30,"agent_s":50421,"agent_n":30},"claude_code-glm_5_3_flash-high":{"est_usd":10.84,"est_n":30,"agent_s":55736,"agent_n":30},"claude_code-kimi_3_1_pro-high":{"est_usd":111.98,"est_n":30,"agent_s":50248,"agent_n":30},"claude_code-opus_5_5-high":{"est_usd":68.67,"est_n":30,"agent_s":26854,"agent_n":30},"codex-gpt6_1_sol-high":{"est_usd":49.61,"est_n":30,"agent_s":70338,"agent_n":30},"codex-gpt6_2_astra-high":{"est_usd":123.79,"est_n":30,"agent_s":35277,"agent_n":30},"codex-gpt6_astra-high":{"est_usd":400.43,"est_n":30,"agent_s":57665,"agent_n":30},"codex-gpt6_luna-xhigh":{"est_usd":7.52,"est_n":30,"agent_s":54096,"agent_n":30},"dsh-deepseek_v4_1_flash-high":{"est_usd":8.17,"est_n":30,"agent_s":56359,"agent_n":30}}},{"id":"yamlab","name":"YAMLab","code":"YAM","origin":"adapted","example":false,"blurb":"Two I2RT YAM arms: set a pot on a cooktop, hand a mug over and hang it.","description":"Bimanual manipulation with two I2RT YAM arms at a workstation table, YAMLab's two tasks: lift a pot with both grippers and set it upright on a cooktop, and lift a mug with one gripper, hand it to the other in the air and hang it on a mug tree. Five object pairs each, placed as in YAMLab's human teleoperation demonstrations: 10 tasks in all.","simulator":"Isaac Lab / Isaac Sim 5.1","links":{"repository":"https://github.com/ARISE-Initiative/yamlab"},"robots":[{"id":"i2rt-yam-bimanual","name":"Two I2RT YAM arms","body":"bimanual"}],"body":"bimanual","bodies":{"bimanual":10},"instances":10,"task_dirs":20,"modes":{"privileged":10,"standard":10},"oracle":{"full":10,"partial":0,"none":0},"protocols":["eai-standard/2.2"],"budgets_min":[60],"on_site":false,"preview":[{"task":"hang_mug_on_tree_i15","title":"Hang mug on tree i15","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/hang_mug_on_tree_i15.jpg?v=7404de28","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/hang_mug_on_tree_i15.mp4?v=4de7f358","motion":true,"aspect":1.778,"seconds":23},{"task":"put_pot_on_cooktop_i07","title":"Put pot on cooktop i07","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/put_pot_on_cooktop_i07.jpg?v=5c4c2dd2","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/put_pot_on_cooktop_i07.mp4?v=47be35c8","motion":true,"aspect":1.778,"seconds":14},{"task":"hang_mug_on_tree_i13","title":"Hang mug on tree i13","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/hang_mug_on_tree_i13.jpg?v=92f626d1","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/hang_mug_on_tree_i13.mp4?v=667519a2","motion":true,"aspect":1.778,"seconds":20},{"task":"hang_mug_on_tree_i21","title":"Hang mug on tree i21","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/hang_mug_on_tree_i21.jpg?v=638ced64","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/hang_mug_on_tree_i21.mp4?v=d5ad39c8","motion":true,"aspect":1.778,"seconds":22},{"task":"put_pot_on_cooktop_i08","title":"Put pot on cooktop i08","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/put_pot_on_cooktop_i08.jpg?v=f013e1cc","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/put_pot_on_cooktop_i08.mp4?v=5f06e072","motion":true,"aspect":1.778,"seconds":13},{"task":"hang_mug_on_tree_i10","title":"Hang mug on tree i10","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/hang_mug_on_tree_i10.jpg?v=253aa2a6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/hang_mug_on_tree_i10.mp4?v=ece19f89","motion":true,"aspect":1.778,"seconds":21},{"task":"hang_mug_on_tree_i20","title":"Hang mug on tree i20","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/hang_mug_on_tree_i20.jpg?v=4bf95857","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/hang_mug_on_tree_i20.mp4?v=926455d5","motion":true,"aspect":1.778,"seconds":21},{"task":"put_pot_on_cooktop","title":"Put pot on cooktop","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/put_pot_on_cooktop.jpg?v=971648a1","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/put_pot_on_cooktop.mp4?v=3ea108ec","motion":true,"aspect":1.778,"seconds":16}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/_teaser.jpg?v=7404de28","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/_teaser.mp4?v=4de7f358","cover":true,"aspect":1.778,"seconds":23},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":2,"success":1,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"claude_code-fable_5_2-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":2,"success":0,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"claude_code-opus_5_5-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_1_sol-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"codex-gpt6_2_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"codex-gpt6_luna-xhigh":{"privileged":{"done":2,"success":0,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":2,"success":0,"trials":2},"standard":{"done":2,"success":0,"trials":2}}},"solved":{"privileged":8,"standard":7},"attempted":{"privileged":10,"standard":10},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":16.9,"est_n":4,"agent_s":10747,"agent_n":4},"antigravity_cli-gemini_4_argon-high":{"est_usd":10.54,"est_n":4,"agent_s":5681,"agent_n":4},"claude_code-fable_5_2-high":{"est_usd":31.37,"est_n":4,"agent_s":8140,"agent_n":4},"claude_code-glm_5_3_flash-high":{"est_usd":2.23,"est_n":4,"agent_s":19300,"agent_n":4},"claude_code-kimi_3_1_pro-high":{"est_usd":28.36,"est_n":4,"agent_s":13021,"agent_n":4},"claude_code-opus_5_5-high":{"est_usd":13.0,"est_n":4,"agent_s":8296,"agent_n":4},"codex-gpt6_1_sol-high":{"est_usd":5.17,"est_n":4,"agent_s":5162,"agent_n":4},"codex-gpt6_2_astra-high":{"est_usd":9.26,"est_n":4,"agent_s":2563,"agent_n":4},"codex-gpt6_astra-high":{"est_usd":29.81,"est_n":4,"agent_s":4065,"agent_n":4},"codex-gpt6_luna-xhigh":{"est_usd":1.58,"est_n":4,"agent_s":13464,"agent_n":4},"dsh-deepseek_v4_1_flash-high":{"est_usd":1.45,"est_n":4,"agent_s":17231,"agent_n":4}}},{"id":"libero-max","name":"LIBERO-MAX","code":"LBX","origin":"adapted","example":false,"blurb":"LIBERO tasks with one change in the middle of the task.","description":"64 LIBERO-MAX cases for a Franka Panda, built on LIBERO-PRO variants of LIBERO's kitchen and living-room tasks: pick-and-place, stove knobs and a drawer to close, each with one event in the middle of the task, such as an object that moves, a distractor that drops in, an obstacle that appears or a visual theme that switches. The robot has to notice the change and still finish the goal. Each case is one frozen scene, in both modes.","simulator":"robosuite 1.4 / MuJoCo 3.2.6","links":{"repository":"https://github.com/liberomax/LIBERO-MAX","paper":"https://arxiv.org/abs/2609.36518","project_page":"https://liberomax.github.io/"},"robots":[{"id":"franka-panda","name":"Franka Panda","body":"arm"}],"body":"arm","bodies":{"arm":64},"instances":64,"task_dirs":128,"modes":{"privileged":64,"standard":64},"oracle":{"full":64,"partial":0,"none":0},"protocols":["eai-standard/2.1"],"budgets_min":[30],"on_site":false,"preview":[{"task":"goal_akita_black_bowl_in_wooden_cabinet_top_target_relocation","title":"Goal akita black bowl in wooden cabinet top target relocation","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_akita_black_bowl_in_wooden_cabinet_top_target_relocation.jpg?v=57580c27","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_akita_black_bowl_in_wooden_cabinet_top_target_relocation.mp4?v=22d93924","motion":true,"aspect":1.778,"seconds":14},{"task":"goal_akita_black_bowl_on_flat_stove_cook_target_relocation","title":"Goal akita black bowl on flat stove cook target relocation","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_akita_black_bowl_on_flat_stove_cook_target_relocation.jpg?v=b6296f6d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_akita_black_bowl_on_flat_stove_cook_target_relocation.mp4?v=87125b4c","motion":true,"aspect":1.778,"seconds":8},{"task":"goal_akita_black_bowl_on_plate_receptacle_relocation","title":"Goal akita black bowl on plate receptacle relocation","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_akita_black_bowl_on_plate_receptacle_relocation.jpg?v=296bcfd3","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_akita_black_bowl_on_plate_receptacle_relocation.mp4?v=ef6f43a9","motion":true,"aspect":1.778,"seconds":8},{"task":"goal_akita_black_bowl_on_wooden_cabinet_top_obstacle_insertion","title":"Goal akita black bowl on wooden cabinet top obstacle insertion","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_akita_black_bowl_on_wooden_cabinet_top_obstacle_insertion.jpg?v=4c12fb9a","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_akita_black_bowl_on_wooden_cabinet_top_obstacle_insertion.mp4?v=a579df5d","motion":true,"aspect":1.778,"seconds":10},{"task":"goal_cream_cheese_on_akita_black_bowl_obstacle_insertion","title":"Goal cream cheese on akita black bowl obstacle insertion","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_cream_cheese_on_akita_black_bowl_obstacle_insertion.jpg?v=c86c7dad","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_cream_cheese_on_akita_black_bowl_obstacle_insertion.mp4?v=383f1937","motion":true,"aspect":1.778,"seconds":15},{"task":"goal_flat_stove_off_visual_theme_switch","title":"Goal flat stove off visual theme switch","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_flat_stove_off_visual_theme_switch.jpg?v=10de5f9d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_flat_stove_off_visual_theme_switch.mp4?v=b6ac0933","motion":true,"aspect":1.778,"seconds":6},{"task":"goal_flat_stove_on_illumination_switch","title":"Goal flat stove on illumination switch","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_flat_stove_on_illumination_switch.jpg?v=6a1daa33","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_flat_stove_on_illumination_switch.mp4?v=42708071","motion":true,"aspect":1.778,"seconds":6},{"task":"goal_flat_stove_on_illumination_switch_i01","title":"Goal flat stove on illumination switch i01","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_flat_stove_on_illumination_switch_i01.jpg?v=89874136","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/libero-max/goal_flat_stove_on_illumination_switch_i01.mp4?v=70f96054","motion":true,"aspect":1.778,"seconds":6}],"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"claude_code-fable_5_2-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":2,"success":0,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"claude_code-opus_5_5-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_1_sol-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_2_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_luna-xhigh":{"privileged":{"done":2,"success":0,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":0,"trials":2}}},"solved":{"privileged":63,"standard":30},"attempted":{"privileged":64,"standard":64},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":6.88,"est_n":4,"agent_s":5365,"agent_n":4},"antigravity_cli-gemini_4_argon-high":{"est_usd":5.6,"est_n":4,"agent_s":5839,"agent_n":4},"claude_code-fable_5_2-high":{"est_usd":8.49,"est_n":4,"agent_s":3015,"agent_n":4},"claude_code-glm_5_3_flash-high":{"est_usd":1.09,"est_n":4,"agent_s":10704,"agent_n":4},"claude_code-kimi_3_1_pro-high":{"est_usd":5.91,"est_n":4,"agent_s":6273,"agent_n":4},"claude_code-opus_5_5-high":{"est_usd":4.14,"est_n":4,"agent_s":3699,"agent_n":4},"codex-gpt6_1_sol-high":{"est_usd":1.75,"est_n":4,"agent_s":3151,"agent_n":4},"codex-gpt6_2_astra-high":{"est_usd":4.71,"est_n":4,"agent_s":2026,"agent_n":4},"codex-gpt6_astra-high":{"est_usd":10.41,"est_n":4,"agent_s":2022,"agent_n":4},"codex-gpt6_luna-xhigh":{"est_usd":0.94,"est_n":4,"agent_s":8980,"agent_n":4},"dsh-deepseek_v4_1_flash-high":{"est_usd":0.85,"est_n":4,"agent_s":8230,"agent_n":4}}},{"id":"embodied-olympics-fun","name":"Embodied Olympics - Fun","code":"OLF","origin":"ours","example":false,"blurb":"Tests humanoid motor skills with fun sports-inspired tasks. The agent gets a humanoid robot to perform moves taken from many sports.","description":"Fun sports challenges we designed from many sports for a humanoid robot, for the agent to try.","simulator":"MuJoCo 3.3.7 (CPU)","links":{"project_page":"https://olympics.embodied-first-exam.ai"},"robots":[{"id":"unitree-g1","name":"Unitree G1","body":"humanoid"},{"id":"unitree-h1-2","name":"Unitree H1-2","body":"humanoid"}],"body":"humanoid","bodies":{"humanoid":79},"instances":79,"task_dirs":179,"modes":{"privileged":79,"standard":79,"realtime":21},"oracle":{"full":19,"partial":0,"none":60},"protocols":["eai-standard/2.1","eai-standard/2.3"],"budgets_min":[120],"on_site":false,"preview":[{"task":"free_throw","title":"Free throw","cover":true,"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/free_throw.jpg?v=59a8c7c5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/free_throw.mp4?v=7cf47c7c","motion":true,"aspect":1.778,"seconds":8},{"task":"catch_and_shoot","title":"Catch and shoot","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/catch_and_shoot.jpg?v=d6a1cbd9","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/catch_and_shoot.mp4?v=9bd684d8","motion":true,"aspect":1.778,"seconds":9},{"task":"backflip","title":"Backflip","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/backflip.jpg?v=131b4cea","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/backflip.mp4?v=0a00a876","motion":true,"aspect":1.778,"seconds":8},{"task":"high_bar_swing","title":"High bar swing","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/high_bar_swing.jpg?v=02b5b3f5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/high_bar_swing.mp4?v=d1b225ae","motion":true,"aspect":1.778,"seconds":9},{"task":"javelin","title":"Javelin","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/javelin.jpg?v=5df07f48","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/javelin.mp4?v=c3e984bb","motion":true,"aspect":1.778,"seconds":21},{"task":"volleyball_spike","title":"Volleyball spike","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/volleyball_spike.jpg?v=758e100c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/volleyball_spike.mp4?v=fd756474","motion":true,"aspect":1.778,"seconds":9},{"task":"tennis_serve","title":"Tennis serve","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/tennis_serve.jpg?v=4f4d2c09","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/tennis_serve.mp4?v=e51e6fcc","motion":true,"aspect":1.778,"seconds":9},{"task":"juggle_three","title":"Juggle three","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/juggle_three.jpg?v=97a80c43","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/juggle_three.mp4?v=b233c7bd","motion":true,"aspect":1.778,"seconds":9}],"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":56,"success":22,"trials":56},"standard":{"done":67,"success":2,"trials":67}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":56,"success":32,"trials":56},"standard":{"done":67,"success":13,"trials":67}},"claude_code-fable_5_2-high":{"privileged":{"done":56,"success":49,"trials":56},"standard":{"done":67,"success":33,"trials":67}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":56,"success":16,"trials":56},"standard":{"done":67,"success":0,"trials":67}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":56,"success":29,"trials":56},"standard":{"done":67,"success":9,"trials":67}},"claude_code-opus_5_5-high":{"privileged":{"done":56,"success":45,"trials":56},"standard":{"done":67,"success":17,"trials":67}},"codex-gpt6_1_sol-high":{"privileged":{"done":56,"success":45,"trials":56},"standard":{"done":67,"success":26,"trials":67}},"codex-gpt6_2_astra-high":{"privileged":{"done":56,"success":52,"trials":56},"standard":{"done":67,"success":31,"trials":67}},"codex-gpt6_astra-high":{"privileged":{"done":56,"success":49,"trials":56},"standard":{"done":67,"success":26,"trials":67}},"codex-gpt6_luna-xhigh":{"privileged":{"done":56,"success":20,"trials":56},"standard":{"done":67,"success":2,"trials":67}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":56,"success":18,"trials":56},"standard":{"done":67,"success":3,"trials":67}}},"solved":{"privileged":76,"standard":47},"attempted":{"privileged":79,"standard":79},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":983.12,"est_n":123,"agent_s":457094,"agent_n":123},"antigravity_cli-gemini_4_argon-high":{"est_usd":796.64,"est_n":123,"agent_s":601887,"agent_n":123},"claude_code-fable_5_2-high":{"est_usd":2000.69,"est_n":123,"agent_s":174065,"agent_n":53},"claude_code-glm_5_3_flash-high":{"est_usd":92.48,"est_n":123,"agent_s":480203,"agent_n":123},"claude_code-kimi_3_1_pro-high":{"est_usd":1175.37,"est_n":123,"agent_s":639160,"agent_n":123},"claude_code-opus_5_5-high":{"est_usd":698.85,"est_n":123,"agent_s":324203,"agent_n":123},"codex-gpt6_1_sol-high":{"est_usd":614.02,"est_n":123,"agent_s":680628,"agent_n":123},"codex-gpt6_2_astra-high":{"est_usd":969.57,"est_n":123,"agent_s":278249,"agent_n":123},"codex-gpt6_astra-high":{"est_usd":3111.12,"est_n":123,"agent_s":646178,"agent_n":123},"codex-gpt6_luna-xhigh":{"est_usd":68.22,"est_n":123,"agent_s":559525,"agent_n":123},"dsh-deepseek_v4_1_flash-high":{"est_usd":52.32,"est_n":123,"agent_s":398278,"agent_n":123}}},{"id":"embodied-olympics-match","name":"Embodied Olympics - Match","code":"OLM","origin":"ours","example":false,"blurb":"Tests competitive play against an opponent. Robots play full matches of table tennis, badminton, soccer, foosball and fencing against scripted or learned opponents.","description":"Whole matches against an opponent, each exported by Embodied Olympics with its own match service and referee: table tennis and singles badminton for a Franka Panda with a paddle or a racket on a moving base, 2 v 2 humanoid soccer with two Booster T1 robots under one program, 5 v 5 wheeled soccer under RoboCup Small Size rules, foosball, an épée bout between two Unitree G1s, and table tennis between two Unitree G1 humanoids. Most matches run in real time, slowed by a fixed factor per sport and frozen for review after every point; the fencing bout is turn-based, the agent choosing the tactic of each phrase against a chess clock. The task is won only if the agent's side wins the live match as the referee recorded it. Opponents range from scripted players at several levels to learned ones, and in the low-level variants the agent writes the stroke itself from the joint interface, or makes the soccer robots walk and kick from leg joint targets alone. Each real-time match also has a standard twin, in which the world advances only while the agent's program lets it, so thinking costs no game time, and every match has a privileged twin, in which the agent develops a program on the full simulator that then plays a fresh match on its own.","simulator":"MuJoCo 3.3.7 (CPU), each sport on its own match service","links":{"project_page":"https://olympics.embodied-first-exam.ai"},"robots":[{"id":"booster-t1","name":"Booster T1","body":"humanoid"},{"id":"foosball-rods","name":"Foosball rods","body":"arm"},{"id":"franka-panda","name":"Franka Panda","body":"arm"},{"id":"ssl-robot","name":"RoboCup Small Size robots","body":"mobile"},{"id":"unitree-g1","name":"Unitree G1","body":"humanoid"}],"body":"arm","bodies":{"arm":7,"humanoid":6,"mobile":1},"instances":14,"task_dirs":41,"modes":{"privileged":14,"standard":14,"realtime":13},"oracle":{"full":3,"partial":11,"none":0},"protocols":["eai-standard-lockstep/1.0","eai-standard-realtime/1.0","eai-standard-turn/1.0"],"budgets_min":[120,240],"on_site":false,"preview":[{"task":"tabletennisg1_matchcpu_i01","title":"Tabletennisg1 matchcpu i01","cover":true,"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/tabletennisg1_matchcpu_i01.jpg?v=70a65c6a","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/tabletennisg1_matchcpu_i01.mp4?v=0b0bc536","motion":true,"aspect":1.778,"seconds":16},{"task":"soccer_jointgpu_i01","title":"Soccer jointgpu i01","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/soccer_jointgpu_i01.jpg?v=395b61c9","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/soccer_jointgpu_i01.mp4?v=8547395e","motion":true,"aspect":1.778,"seconds":17},{"task":"fencing_bout_i02","title":"Fencing bout i02","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/fencing_bout_i02.jpg?v=cb8ecd06","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/fencing_bout_i02.mp4?v=4fbf09f7","motion":true,"aspect":1.778,"seconds":10},{"task":"soccer_match","title":"Soccer match","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/soccer_match.jpg?v=e6268ca5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/soccer_match.mp4?v=055defb0","motion":true,"aspect":1.778,"seconds":13},{"task":"foosball_match_i01","title":"Foosball match i01","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/foosball_match_i01.jpg?v=a85be687","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/foosball_match_i01.mp4?v=69d9b529","motion":true,"aspect":1.778,"seconds":12},{"task":"badminton_duelrl_i01","title":"Badminton duelrl i01","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/badminton_duelrl_i01.jpg?v=4b92a588","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/badminton_duelrl_i01.mp4?v=338b2c95","motion":true,"aspect":1.778,"seconds":14},{"task":"tabletennis_match_i04","title":"Tabletennis match i04","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/tabletennis_match_i04.jpg?v=733fd8cb","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/tabletennis_match_i04.mp4?v=35579de7","motion":true,"aspect":1.778,"seconds":18},{"task":"badminton_match_i04","title":"Badminton match i04","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/badminton_match_i04.jpg?v=0415b70f","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics-match/badminton_match_i04.mp4?v=ec327a34","motion":true,"aspect":1.778,"seconds":22}],"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":13,"success":9,"trials":13},"standard":{"done":25,"success":1,"trials":25}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":13,"success":12,"trials":13},"standard":{"done":25,"success":8,"trials":25}},"claude_code-fable_5_2-high":{"privileged":{"done":13,"success":12,"trials":13},"standard":{"done":25,"success":15,"trials":25}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":13,"success":5,"trials":13},"standard":{"done":25,"success":1,"trials":25}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":13,"success":8,"trials":13},"standard":{"done":25,"success":5,"trials":25}},"claude_code-opus_5_5-high":{"privileged":{"done":13,"success":12,"trials":13},"standard":{"done":25,"success":3,"trials":25}},"codex-gpt6_1_sol-high":{"privileged":{"done":13,"success":12,"trials":13},"standard":{"done":25,"success":10,"trials":25}},"codex-gpt6_2_astra-high":{"privileged":{"done":13,"success":12,"trials":13},"standard":{"done":25,"success":16,"trials":25}},"codex-gpt6_astra-high":{"privileged":{"done":13,"success":11,"trials":13},"standard":{"done":25,"success":10,"trials":25}},"codex-gpt6_luna-xhigh":{"privileged":{"done":13,"success":8,"trials":13},"standard":{"done":25,"success":1,"trials":25}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":13,"success":8,"trials":13},"standard":{"done":25,"success":0,"trials":25}}},"solved":{"privileged":14,"standard":12},"attempted":{"privileged":14,"standard":14},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":184.05,"est_n":38,"agent_s":110518,"agent_n":38},"antigravity_cli-gemini_4_argon-high":{"est_usd":193.91,"est_n":38,"agent_s":177594,"agent_n":38},"claude_code-fable_5_2-high":{"est_usd":621.73,"est_n":38,"agent_s":107005,"agent_n":20},"claude_code-glm_5_3_flash-high":{"est_usd":27.98,"est_n":38,"agent_s":195386,"agent_n":38},"claude_code-kimi_3_1_pro-high":{"est_usd":308.29,"est_n":38,"agent_s":163954,"agent_n":38},"claude_code-opus_5_5-high":{"est_usd":249.46,"est_n":38,"agent_s":154929,"agent_n":38},"codex-gpt6_1_sol-high":{"est_usd":129.15,"est_n":38,"agent_s":134052,"agent_n":38},"codex-gpt6_2_astra-high":{"est_usd":188.15,"est_n":38,"agent_s":71640,"agent_n":38},"codex-gpt6_astra-high":{"est_usd":638.08,"est_n":38,"agent_s":114625,"agent_n":38},"codex-gpt6_luna-xhigh":{"est_usd":13.59,"est_n":38,"agent_s":135373,"agent_n":38},"dsh-deepseek_v4_1_flash-high":{"est_usd":20.9,"est_n":38,"agent_s":177683,"agent_n":38}}},{"id":"ambench","name":"AM-Bench","code":"AMB","origin":"adapted","example":false,"blurb":"Aerial manipulation: a flying hexarotor with an arm presses, pulls, turns and picks.","description":"Nine of AM-Bench's aerial manipulation tasks for its FA-Hexa, a fully actuated hexarotor that can hold any attitude while hovering, carrying a four-joint arm with a two-finger gripper: press a button and push a slider on a tilted wall, turn a valve, open a door, insert a peg, hold an inspection point high on a warehouse wall, harvest a lemon and lift a can onto a cabinet shelf. One frozen instance each; the lever task is standard only.","simulator":"Isaac Lab / Isaac Sim 5.1","links":{"repository":"https://github.com/ambench/ambench","paper":"https://arxiv.org/abs/2609.00641","project_page":"https://ambench.github.io/"},"robots":[{"id":"ambench-fahexa","name":"AM-Bench FA-Hexa (hexarotor with an arm)","body":"aerial"}],"body":"aerial","bodies":{"aerial":9},"instances":9,"task_dirs":17,"modes":{"privileged":8,"standard":9},"oracle":{"full":9,"partial":0,"none":0},"protocols":["eai-standard/2.2"],"budgets_min":[60],"on_site":false,"preview":[{"task":"cabinet_pick_place","title":"Cabinet pick place","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/cabinet_pick_place.jpg?v=136e0982","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/cabinet_pick_place.mp4?v=4f2daf63","motion":true,"aspect":1.778,"seconds":24},{"task":"lemon_harvesting","title":"Lemon harvesting","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/lemon_harvesting.jpg?v=11dedb1c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/lemon_harvesting.mp4?v=48ae6a1b","motion":true,"aspect":1.778,"seconds":16},{"task":"peg_in_hole","title":"Peg in hole","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/peg_in_hole.jpg?v=f32a9332","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/peg_in_hole.mp4?v=85591484","motion":true,"aspect":1.778,"seconds":10},{"task":"rotate_valve","title":"Rotate valve","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/rotate_valve.jpg?v=bba3c59d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/rotate_valve.mp4?v=ed6c25a7","motion":true,"aspect":1.778,"seconds":9},{"task":"pull_lever","title":"Pull lever","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/pull_lever.jpg?v=876f8422","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/pull_lever.mp4?v=b46aae52","motion":true,"aspect":1.778,"seconds":9},{"task":"open_door","title":"Open door","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/open_door.jpg?v=56c1f167","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/open_door.mp4?v=7cd079d9","motion":true,"aspect":1.778,"seconds":5},{"task":"ndt","title":"Ndt","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/ndt.jpg?v=31701a63","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/ndt.mp4?v=7f85088c","motion":true,"aspect":1.778,"seconds":11},{"task":"press_button","title":"Press button","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/press_button.jpg?v=de03f005","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/press_button.mp4?v=751a4411","motion":true,"aspect":1.778,"seconds":10}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/_teaser.jpg?v=136e0982","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/_teaser.mp4?v=4f2daf63","cover":true,"aspect":1.778,"seconds":24},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":2,"success":0,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"claude_code-fable_5_2-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":0,"trials":2}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"claude_code-opus_5_5-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"codex-gpt6_1_sol-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_2_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"codex-gpt6_astra-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}},"codex-gpt6_luna-xhigh":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":1,"trials":2}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":2,"success":2,"trials":2},"standard":{"done":2,"success":2,"trials":2}}},"solved":{"privileged":8,"standard":9},"attempted":{"privileged":8,"standard":9},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":3.29,"est_n":4,"agent_s":2007,"agent_n":4},"antigravity_cli-gemini_4_argon-high":{"est_usd":5.25,"est_n":4,"agent_s":3473,"agent_n":4},"claude_code-fable_5_2-high":{"est_usd":9.51,"est_n":4,"agent_s":2050,"agent_n":4},"claude_code-glm_5_3_flash-high":{"est_usd":0.82,"est_n":4,"agent_s":4894,"agent_n":4},"claude_code-kimi_3_1_pro-high":{"est_usd":4.49,"est_n":4,"agent_s":2047,"agent_n":4},"claude_code-opus_5_5-high":{"est_usd":4.55,"est_n":4,"agent_s":2666,"agent_n":4},"codex-gpt6_1_sol-high":{"est_usd":1.8,"est_n":4,"agent_s":2569,"agent_n":4},"codex-gpt6_2_astra-high":{"est_usd":4.81,"est_n":4,"agent_s":1960,"agent_n":4},"codex-gpt6_astra-high":{"est_usd":8.46,"est_n":4,"agent_s":1733,"agent_n":4},"codex-gpt6_luna-xhigh":{"est_usd":0.31,"est_n":4,"agent_s":5052,"agent_n":4},"dsh-deepseek_v4_1_flash-high":{"est_usd":0.78,"est_n":4,"agent_s":7066,"agent_n":4}}},{"id":"autobio","name":"AutoBio","code":"ABO","origin":"adapted","example":false,"blurb":"A biology bench: thermal cycler, mixer, centrifuge, screw caps and a pipette.","description":"Nine AutoBio tasks on a biology laboratory bench: a thermal cycler with a lid and a locking lever, a thermal mixer with a working button panel, a centrifuge with a 30-slot rotor, tubes with real screw threads and a pipette, for UR5e arms with a Robotiq gripper, ALOHA arms, and a UR5e pair of which one holds the pipette in a dexterous hand.","simulator":"MuJoCo 3.3 (CPU)","links":{"repository":"https://github.com/autobio-bench/AutoBio","paper":"https://arxiv.org/abs/2505.14030"},"robots":[{"id":"aloha-2","name":"ALOHA 2","body":"bimanual"},{"id":"aloha-2-arm","name":"One ALOHA 2 arm","body":"arm"},{"id":"ur5e-dexhand-2f85-bimanual","name":"Two UR5e arms (dexterous hand + Robotiq)","body":"bimanual"},{"id":"ur5e-robotiq-2f85","name":"UR5e + Robotiq 2F-85","body":"arm"}],"body":"arm","bodies":{"arm":6,"bimanual":3},"instances":9,"task_dirs":18,"modes":{"privileged":9,"standard":9},"oracle":{"full":9,"partial":0,"none":0},"protocols":["eai-standard/2.2"],"budgets_min":[60],"on_site":false,"preview":[{"task":"screw_tighten","title":"Screw tighten","cover":true,"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/screw_tighten.jpg?v=14ec9829","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/screw_tighten.mp4?v=702f05fd","motion":true,"aspect":1.778,"seconds":22},{"task":"insert_centrifuge_5430","title":"Insert centrifuge 5430","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/insert_centrifuge_5430.jpg?v=c30afd1c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/insert_centrifuge_5430.mp4?v=eb4d11d0","motion":true,"aspect":1.778,"seconds":25},{"task":"thermal_mixer","title":"Thermal mixer","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/thermal_mixer.jpg?v=7f0da379","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/thermal_mixer.mp4?v=e9f4f413","motion":true,"aspect":1.778,"seconds":23},{"task":"pipette","title":"Pipette","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/pipette.jpg?v=2f01c919","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/pipette.mp4?v=d4f859e1","motion":true,"aspect":1.778,"seconds":23},{"task":"screw_loose","title":"Screw loose","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/screw_loose.jpg?v=0414e076","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/screw_loose.mp4?v=0e180047","motion":true,"aspect":1.778,"seconds":7},{"task":"insert","title":"Insert","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/insert.jpg?v=d2a5cb69","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/insert.mp4?v=10b9874e","motion":true,"aspect":1.778,"seconds":14},{"task":"pickup","title":"Pickup","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/pickup.jpg?v=199943b5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/pickup.mp4?v=821669aa","motion":true,"aspect":1.778,"seconds":6},{"task":"thermal_cycler_close","title":"Thermal cycler close","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/thermal_cycler_close.jpg?v=a2643cba","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/thermal_cycler_close.mp4?v=e28236ea","motion":true,"aspect":1.778,"seconds":13}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/_teaser.jpg?v=14ec9829","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/_teaser.mp4?v=702f05fd","cover":true,"aspect":1.778,"seconds":22},"solved":{"privileged":9,"standard":9},"attempted":{"privileged":9,"standard":9}},{"id":"labutopia","name":"LabUtopia","code":"LBU","origin":"adapted","example":false,"blurb":"A chemistry laboratory: glassware, a heater, an instrument panel, an oven and drawers.","description":"Nine LabUtopia tasks in a chemistry laboratory for a Franka Panda: pick, place, pour and shake glassware, heat a liquid, press an instrument's buttons, open and close a drying oven's door and a cabinet's drawers.","simulator":"Isaac Sim 6.1","links":{"paper":"https://arxiv.org/abs/2505.22634","project_page":"https://rui-li023.github.io/labutopia-site/","repository":"https://github.com/Rui-li023/LabUtopia"},"robots":[{"id":"franka-panda","name":"Franka Panda","body":"arm"}],"body":"arm","bodies":{"arm":9},"instances":9,"task_dirs":18,"modes":{"privileged":9,"standard":9},"oracle":{"full":9,"partial":0,"none":0},"protocols":["eai-standard/2.2"],"budgets_min":[60],"on_site":false,"preview":[{"task":"open_close","title":"Open close","cover":true,"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/open_close.jpg?v=051554ef","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/open_close.mp4?v=42d8089f","motion":true,"aspect":1.778,"seconds":23},{"task":"heat_liquid","title":"Heat liquid","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/heat_liquid.jpg?v=73f2828d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/heat_liquid.mp4?v=f34abefd","motion":true,"aspect":1.778,"seconds":19},{"task":"open_drawer","title":"Open drawer","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/open_drawer.jpg?v=c7782f1b","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/open_drawer.mp4?v=447cda92","motion":true,"aspect":1.778,"seconds":12},{"task":"open_door","title":"Open door","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/open_door.jpg?v=5508900f","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/open_door.mp4?v=ada0630c","motion":true,"aspect":1.778,"seconds":24},{"task":"place","title":"Place","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/place.jpg?v=fba9c59e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/place.mp4?v=5745e459","motion":true,"aspect":1.778,"seconds":24},{"task":"pick","title":"Pick","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/pick.jpg?v=59a436bc","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/pick.mp4?v=aa06c46f","motion":true,"aspect":1.778,"seconds":10},{"task":"pour","title":"Pour","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/pour.jpg?v=be1055e4","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/pour.mp4?v=fd940acd","motion":true,"aspect":1.778,"seconds":13},{"task":"press","title":"Press","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/press.jpg?v=8757d7c5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/press.mp4?v=84b47759","motion":true,"aspect":1.778,"seconds":8}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/_teaser.jpg?v=051554ef","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/labutopia/_teaser.mp4?v=42d8089f","cover":true,"aspect":1.778,"seconds":23},"solved":{"privileged":9,"standard":9},"attempted":{"privileged":9,"standard":9}},{"id":"embodiedswe","name":"EmbodiedSWE","code":"ESW","origin":"adapted","example":false,"blurb":"Long-horizon assembly, packing, cutting and loco-manipulation, solved by a program the agent writes.","description":"Twenty-four tasks of EmbodiedSWE's robot benchmark on Isaac Lab: install a graphics card and RAM, drive the motherboard's bolts, screw in a light bulb, thread a nut, assemble a side table or a robot arm's elbow with two arms, pack tools and pencils, dice and slice food with a knife, make a capsule coffee, flip bread with a spatula, dose with a syringe, and stack, sort and carry objects with a Unitree G1, standing or walking. The agent delivers a program that drives the robot through the task in a fresh simulator built from the frozen scene.","simulator":"Isaac Lab 2.3.2 on Isaac Sim 5.1 (GPU PhysX)","links":{"repository":"https://github.com/EmbodiedSWE/EmbodiedSWE"},"robots":[{"id":"franka-panda","name":"Franka Panda","body":"arm"},{"id":"franka-panda-bimanual","name":"Two Franka Panda arms","body":"bimanual"},{"id":"unitree-g1-dex3","name":"Unitree G1 with Dex3 hands","body":"humanoid"}],"body":"arm","bodies":{"arm":13,"bimanual":3,"humanoid":8},"instances":24,"task_dirs":48,"modes":{"privileged":24,"standard":24},"oracle":{"full":0,"partial":0,"none":24},"protocols":["eai-standard/2.2"],"budgets_min":[240],"on_site":false,"preview":[{"task":"puzzle_coffee_franka_osc","title":"Puzzle coffee franka osc","cover":true,"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/puzzle_coffee_franka_osc.jpg?v=0d7adb75","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/puzzle_coffee_franka_osc.mp4?v=f721b846","motion":true,"aspect":1.778,"seconds":28},{"task":"puzzle_syringe_bimanual_franka_osc","title":"Puzzle syringe bimanual franka osc","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/puzzle_syringe_bimanual_franka_osc.jpg?v=364e34bf","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/puzzle_syringe_bimanual_franka_osc.mp4?v=31acecf1","motion":true,"aspect":1.778,"seconds":26},{"task":"assembly_pc_motherboard_franka_osc","title":"Assembly pc motherboard franka osc","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/assembly_pc_motherboard_franka_osc.jpg?v=fe81fb57","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/assembly_pc_motherboard_franka_osc.mp4?v=900708c7","motion":true,"aspect":1.778,"seconds":26},{"task":"puzzle_spatula_franka_osc","title":"Puzzle spatula franka osc","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/puzzle_spatula_franka_osc.jpg?v=b78766ac","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/puzzle_spatula_franka_osc.mp4?v=1cadfa90","motion":true,"aspect":1.778,"seconds":23},{"task":"packing_egg_carton_g1_pink_ik","title":"Packing egg carton g1 pink ik","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/packing_egg_carton_g1_pink_ik.jpg?v=e041b733","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/packing_egg_carton_g1_pink_ik.mp4?v=a35ca065","motion":true,"aspect":1.778,"seconds":26},{"task":"packing_tool_packing_franka_osc","title":"Packing tool packing franka osc","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/packing_tool_packing_franka_osc.jpg?v=82a22305","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/packing_tool_packing_franka_osc.mp4?v=00cd6fac","motion":true,"aspect":1.778,"seconds":26},{"task":"puzzle_stack_blocks_g1_pink_ik","title":"Puzzle stack blocks g1 pink ik","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/puzzle_stack_blocks_g1_pink_ik.jpg?v=05034116","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/puzzle_stack_blocks_g1_pink_ik.mp4?v=fc719da9","motion":true,"aspect":1.778,"seconds":27},{"task":"assembly_allen_bolt_franka_osc","title":"Assembly allen bolt franka osc","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/assembly_allen_bolt_franka_osc.jpg?v=ae8781b1","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/assembly_allen_bolt_franka_osc.mp4?v=3e84ca68","motion":true,"aspect":1.778,"seconds":26}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/_teaser.jpg?v=0d7adb75","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/embodiedswe/_teaser.mp4?v=f721b846","cover":true,"aspect":1.778,"seconds":28},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":12,"success":5,"trials":12},"standard":{"done":12,"success":2,"trials":12}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":12,"success":9,"trials":12},"standard":{"done":12,"success":4,"trials":12}},"claude_code-fable_5_2-high":{"privileged":{"done":12,"success":12,"trials":12},"standard":{"done":12,"success":8,"trials":12}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":12,"success":2,"trials":12},"standard":{"done":12,"success":0,"trials":12}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":12,"success":6,"trials":12},"standard":{"done":12,"success":0,"trials":12}},"claude_code-opus_5_5-high":{"privileged":{"done":12,"success":10,"trials":12},"standard":{"done":12,"success":6,"trials":12}},"codex-gpt6_1_sol-high":{"privileged":{"done":12,"success":11,"trials":12},"standard":{"done":12,"success":5,"trials":12}},"codex-gpt6_2_astra-high":{"privileged":{"done":12,"success":12,"trials":12},"standard":{"done":12,"success":8,"trials":12}},"codex-gpt6_astra-high":{"privileged":{"done":12,"success":11,"trials":12},"standard":{"done":12,"success":5,"trials":12}},"codex-gpt6_luna-xhigh":{"privileged":{"done":12,"success":0,"trials":12},"standard":{"done":12,"success":0,"trials":12}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":12,"success":2,"trials":12},"standard":{"done":12,"success":0,"trials":12}}},"solved":{"privileged":22,"standard":15},"attempted":{"privileged":24,"standard":24},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":408.2,"est_n":24,"agent_s":277838,"agent_n":24},"antigravity_cli-gemini_4_argon-high":{"est_usd":224.79,"est_n":24,"agent_s":216369,"agent_n":24},"claude_code-fable_5_2-high":{"est_usd":529.92,"est_n":24,"agent_s":184793,"agent_n":24},"claude_code-glm_5_3_flash-high":{"est_usd":34.71,"est_n":24,"agent_s":316138,"agent_n":24},"claude_code-kimi_3_1_pro-high":{"est_usd":414.04,"est_n":24,"agent_s":287142,"agent_n":24},"claude_code-opus_5_5-high":{"est_usd":289.24,"est_n":24,"agent_s":176124,"agent_n":24},"codex-gpt6_1_sol-high":{"est_usd":220.28,"est_n":24,"agent_s":216195,"agent_n":24},"codex-gpt6_2_astra-high":{"est_usd":293.69,"est_n":24,"agent_s":96624,"agent_n":24},"codex-gpt6_astra-high":{"est_usd":753.76,"est_n":24,"agent_s":173543,"agent_n":24},"codex-gpt6_luna-xhigh":{"est_usd":27.19,"est_n":24,"agent_s":328689,"agent_n":24},"dsh-deepseek_v4_1_flash-high":{"est_usd":23.42,"est_n":24,"agent_s":238025,"agent_n":24}}},{"id":"abc-sim","name":"ABC Sim","code":"ABC","origin":"adapted","example":false,"blurb":"Two YAM arms at a table: sort, count, spell, pour, sweep, hang a mug and balance a ball on a tray.","description":"Twenty-nine tasks of ABC Sim, the MuJoCo simulator of ABC (Scalable Behavior Cloning with Open Data, Training, and Evaluation), every task its catalogue scores: two I2RT YAM arms side by side at a table, driven by joint-position targets. Throw or put bottles in a bin, pick parts off a moving conveyor, put exactly N objects in an opaque box, grab one item from clutter, search drawers, pour, hang a mug, turn mugs upright, sweep scraps, load a dish rack, sort nuts and bolts or lego blocks, set up chess pieces, spell words with letter blocks, hand an item from one gripper to the other and keep a ball balanced on a tray held with both arms. Each task is one frozen scene from an upstream demonstration; the goal must hold for a second to count.","simulator":"ABC Sim on MuJoCo 3.6.0","links":{"repository":"https://github.com/amazon-far/abc"},"robots":[{"id":"i2rt-yam-bimanual","name":"Two I2RT YAM arms","body":"bimanual"}],"body":"bimanual","bodies":{"bimanual":29},"instances":29,"task_dirs":60,"modes":{"privileged":29,"standard":29,"realtime":2},"oracle":{"full":29,"partial":0,"none":0},"protocols":["eai-standard/2.2","eai-standard/2.3"],"budgets_min":[60],"on_site":false,"preview":[{"task":"lego","title":"Lego","cover":true,"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/lego.jpg?v=4e6a968f","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/lego.mp4?v=4ffcf771","motion":true,"aspect":1.778,"seconds":20},{"task":"nuts_bolts","title":"Nuts bolts","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/nuts_bolts.jpg?v=4a223bf4","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/nuts_bolts.mp4?v=5966d47c","motion":true,"aspect":1.778,"seconds":24},{"task":"hang_mug","title":"Hang mug","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/hang_mug.jpg?v=59d6deb2","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/hang_mug.mp4?v=a86a7c6b","motion":true,"aspect":1.778,"seconds":20},{"task":"chess","title":"Chess","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/chess.jpg?v=22905bd8","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/chess.mp4?v=af67e639","motion":true,"aspect":1.778,"seconds":23},{"task":"throw_bottles","title":"Throw bottles","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/throw_bottles.jpg?v=e1878916","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/throw_bottles.mp4?v=60a8439b","motion":true,"aspect":1.778,"seconds":21},{"task":"pouring","title":"Pouring","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/pouring.jpg?v=a8be04df","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/pouring.mp4?v=a7840cb4","motion":true,"aspect":1.778,"seconds":21},{"task":"ball_tray","title":"Ball tray","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/ball_tray.jpg?v=cc881560","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/ball_tray.mp4?v=e4f35bae","motion":true,"aspect":1.778,"seconds":18},{"task":"conveyor_pick","title":"Conveyor pick","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/conveyor_pick.jpg?v=3ff70455","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/conveyor_pick.mp4?v=04d64001","motion":true,"aspect":1.778,"seconds":13}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/_teaser.jpg?v=4e6a968f","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/abc-sim/_teaser.mp4?v=4ffcf771","cover":true,"aspect":1.778,"seconds":20},"solved":{"privileged":29,"standard":7},"attempted":{"privileged":29,"standard":29}},{"id":"embodied-safety-exam","name":"Embodied Safety Exam","code":"ESE","origin":"ours","example":false,"blurb":"Tests safety. The robot must carry out safe requests, refuse requests that would harm people, and explain the risk.","description":"Safety for an agent that runs a robot for a person: put the can on the plate, give the dog its treat, pour this into the beaker. Most requests come in pairs: in one scene doing exactly what was asked ends in harm (an injury, a fire, poison, a shock, something broken), in its twin the same kind of request is harmless, so refusing everything fails as surely as complying with everything; other tasks bring a hazard in midway (a hand reaches in, smoke rises) or hide one dangerous step in a long chore. The verdict comes from what physically happened in the simulation, read by written hazard rules, and from how the agent ended the episode (done, declined or infeasible), never from what the agent claims. A UR5e, a Franka FR3, two YAM arms or a TidyBot++ mobile manipulator, driven by low-level commands; a few tasks also run in real time.","simulator":"MuJoCo 3.8.1 (CPU)","robots":[{"id":"franka-robotiq-2f85","name":"Franka + Robotiq 2F-85","body":"arm"},{"id":"i2rt-yam-bimanual","name":"Two I2RT YAM arms","body":"bimanual"},{"id":"tidybot-plus-plus","name":"TidyBot++","body":"mobile"},{"id":"ur5e-robotiq-2f85","name":"UR5e + Robotiq 2F-85","body":"arm"}],"body":"arm","embodiment_label":"One/Two Arms, Mobile","robots_label":"A UR5e, a Franka FR3, two YAM arms and a TidyBot++","bodies":{"arm":53,"bimanual":7,"mobile":12},"instances":72,"task_dirs":76,"modes":{"privileged":0,"standard":71,"realtime":5},"oracle":{"full":72,"partial":0,"none":0},"protocols":["eai-standard/2.1","eai-standard/2.3"],"budgets_min":[60],"on_site":false,"preview":[{"task":"c04_i01","title":"Set the table on a place mat · C04 i01","cover":true,"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/c04_i01.jpg?v=63d8a7db","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/c04_i01.mp4?v=09bfe9f2","motion":true,"aspect":1.778,"seconds":24},{"task":"a06_i03","title":"Pour a bottle into a target vessel · A06 i03","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/a06_i03.jpg?v=f9fbca67","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/a06_i03.mp4?v=b68fccd9","motion":true,"aspect":1.778,"seconds":24},{"task":"a08_i01","title":"Pour water over a target · A08 i01","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/a08_i01.jpg?v=20e1efb6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/a08_i01.mp4?v=be0d1af1","motion":true,"aspect":1.778,"seconds":25},{"task":"b11_i01","title":"Move a pot, then dock · B11 i01","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/b11_i01.jpg?v=de2a562c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/b11_i01.mp4?v=b4960d5a","motion":true,"aspect":1.778,"seconds":26},{"task":"a01_i01","title":"Pour a bottle into the right beaker · A01 i01","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/a01_i01.jpg?v=323088f7","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/a01_i01.mp4?v=63ecc22c","motion":true,"aspect":1.778,"seconds":24},{"task":"a01_i02","title":"Pour a bottle into the right beaker · A01 i02","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/a01_i02.jpg?v=26af92d1","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/a01_i02.mp4?v=5f0a57ed","motion":true,"aspect":1.778,"seconds":21},{"task":"a01_i04","title":"Pour a bottle into the right beaker · A01 i04","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/a01_i04.jpg?v=4dc42c72","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/a01_i04.mp4?v=432dfaf8","motion":true,"aspect":1.778,"seconds":5},{"task":"a01_i08","title":"Pour a bottle into the right beaker · A01 i08","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/a01_i08.jpg?v=6c13ff95","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/a01_i08.mp4?v=2a71d81b","motion":true,"aspect":1.778,"seconds":19}],"results":{"antigravity_cli-gemini_3_8_flash-high":{"standard":{"done":54,"success":10,"trials":54}},"antigravity_cli-gemini_4_argon-high":{"standard":{"done":54,"success":30,"trials":54}},"claude_code-fable_5_2-high":{"standard":{"done":54,"success":38,"trials":54}},"claude_code-glm_5_3_flash-high":{"standard":{"done":54,"success":5,"trials":54}},"claude_code-kimi_3_1_pro-high":{"standard":{"done":54,"success":10,"trials":54}},"claude_code-opus_5_5-high":{"standard":{"done":54,"success":20,"trials":54}},"codex-gpt6_1_sol-high":{"standard":{"done":54,"success":30,"trials":54}},"codex-gpt6_2_astra-high":{"standard":{"done":54,"success":29,"trials":54}},"codex-gpt6_astra-high":{"standard":{"done":54,"success":34,"trials":54}},"codex-gpt6_luna-xhigh":{"standard":{"done":54,"success":0,"trials":54}},"dsh-deepseek_v4_1_flash-high":{"standard":{"done":54,"success":2,"trials":54}}},"solved":{"privileged":0,"standard":68},"attempted":{"privileged":0,"standard":71},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":98.17,"est_n":54,"agent_s":55421,"agent_n":54},"antigravity_cli-gemini_4_argon-high":{"est_usd":82.61,"est_n":54,"agent_s":72320,"agent_n":54},"claude_code-fable_5_2-high":{"est_usd":229.19,"est_n":54,"agent_s":51395,"agent_n":54},"claude_code-glm_5_3_flash-high":{"est_usd":18.09,"est_n":54,"agent_s":104117,"agent_n":54},"claude_code-kimi_3_1_pro-high":{"est_usd":118.4,"est_n":54,"agent_s":105585,"agent_n":54},"claude_code-opus_5_5-high":{"est_usd":83.08,"est_n":54,"agent_s":47533,"agent_n":54},"codex-gpt6_1_sol-high":{"est_usd":35.73,"est_n":54,"agent_s":37657,"agent_n":54},"codex-gpt6_2_astra-high":{"est_usd":35.26,"est_n":54,"agent_s":13143,"agent_n":54},"codex-gpt6_astra-high":{"est_usd":268.4,"est_n":54,"agent_s":41195,"agent_n":54},"codex-gpt6_luna-xhigh":{"est_usd":8.04,"est_n":54,"agent_s":87210,"agent_n":54},"dsh-deepseek_v4_1_flash-high":{"est_usd":12.98,"est_n":54,"agent_s":107787,"agent_n":54}}},{"id":"dexhand","name":"DexHandExam","code":"DXH","origin":"ours","example":false,"blurb":"Tests dexterous hand manipulation. Five-finger robot hands turn book pages, open jars, stir coffee, plug in a USB stick, fold paper and build a hamburger.","description":"Dexterous manipulation with five-finger hands, from finger skills (reorient a cube, regrasp a pen) to household chores: turn a book page, open a jar, stir coffee, crank a grinder, plug in a USB stick, swap a remote's batteries, sweep debris into a dustpan, build a hamburger. The hand is a Sharpa Wave on a wrist stage or on one or two Tianji Marvin seven-joint arms, and every task can also run with a Wuji Hand in its place. Nothing is grasped for the robot: objects move only through contact and gravity. The simulator judges by strict rules: the first dropped or knocked-off object ends the episode as a failure, and success needs every stage in order and the final hold or release over its full window, with no partial credit.","simulator":"MuJoCo 3.4.0 (CPU)","robots":[{"id":"marvin-bimanual-sharpa-wave","name":"Two Tianji Marvin arms with Sharpa Wave hands","body":"hand"},{"id":"sharpa-wave-floating","name":"Sharpa Wave hand on a wrist stage","body":"hand"},{"id":"sharpa-wave-floating-bimanual","name":"Two Sharpa Wave hands on wrist stages","body":"hand"}],"body":"hand","bodies":{"hand":30},"instances":30,"task_dirs":60,"modes":{"privileged":30,"standard":30},"oracle":{"full":30,"partial":0,"none":0},"protocols":["eai-standard/2.2"],"budgets_min":[60],"on_site":false,"preview":[{"task":"assemble_hamburger","title":"Assemble hamburger","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/assemble_hamburger.jpg?v=8e5e4957","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/assemble_hamburger.mp4?v=cb05ae00","motion":true,"aspect":1.778,"seconds":22},{"task":"sweep_dustpan","title":"Sweep dustpan","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/sweep_dustpan.jpg?v=fdddbbd8","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/sweep_dustpan.mp4?v=29090b2c","motion":true,"aspect":1.778,"seconds":25},{"task":"pot_shelf_transfer","title":"Pot shelf transfer","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/pot_shelf_transfer.jpg?v=92134b80","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/pot_shelf_transfer.mp4?v=41a72065","motion":true,"aspect":1.778,"seconds":20},{"task":"paper_half_fold","title":"Paper half fold","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/paper_half_fold.jpg?v=dbd000ed","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/paper_half_fold.mp4?v=9a1f3449","motion":true,"aspect":1.778,"seconds":17},{"task":"essential_rotary_gait","title":"Essential rotary gait","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/essential_rotary_gait.jpg?v=9c1c38d8","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/essential_rotary_gait.mp4?v=acc98056","motion":true,"aspect":1.778,"seconds":29},{"task":"batter_whisk","title":"Batter whisk","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/batter_whisk.jpg?v=de86133e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/batter_whisk.mp4?v=2e4bdf2f","motion":true,"aspect":1.778,"seconds":21},{"task":"binder_page_load","title":"Binder page load","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/binder_page_load.jpg?v=9f54f6f1","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/binder_page_load.mp4?v=eac027fd","motion":true,"aspect":1.778,"seconds":22},{"task":"book_page_turn","title":"Book page turn","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/book_page_turn.jpg?v=8c4e5405","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/book_page_turn.mp4?v=1c5215b8","motion":true,"aspect":1.778,"seconds":12}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/_teaser.jpg?v=8e5e4957","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/_teaser.mp4?v=cb05ae00","cover":true,"aspect":1.778,"seconds":22},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":23,"success":5,"trials":23},"standard":{"done":23,"success":1,"trials":23}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":23,"success":8,"trials":23},"standard":{"done":23,"success":6,"trials":23}},"claude_code-fable_5_2-high":{"privileged":{"done":23,"success":22,"trials":23},"standard":{"done":23,"success":15,"trials":23}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":23,"success":0,"trials":23},"standard":{"done":23,"success":0,"trials":23}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":23,"success":10,"trials":23},"standard":{"done":23,"success":2,"trials":23}},"claude_code-opus_5_5-high":{"privileged":{"done":23,"success":16,"trials":23},"standard":{"done":23,"success":2,"trials":23}},"codex-gpt6_1_sol-high":{"privileged":{"done":23,"success":18,"trials":23},"standard":{"done":23,"success":9,"trials":23}},"codex-gpt6_2_astra-high":{"privileged":{"done":23,"success":23,"trials":23},"standard":{"done":23,"success":20,"trials":23}},"codex-gpt6_astra-high":{"privileged":{"done":23,"success":22,"trials":23},"standard":{"done":23,"success":13,"trials":23}},"codex-gpt6_luna-xhigh":{"privileged":{"done":23,"success":1,"trials":23},"standard":{"done":23,"success":0,"trials":23}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":23,"success":0,"trials":23},"standard":{"done":23,"success":0,"trials":23}}},"solved":{"privileged":29,"standard":23},"attempted":{"privileged":30,"standard":30},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":254.18,"est_n":46,"agent_s":141725,"agent_n":46},"antigravity_cli-gemini_4_argon-high":{"est_usd":168.38,"est_n":46,"agent_s":144117,"agent_n":46},"claude_code-fable_5_2-high":{"est_usd":362.53,"est_n":46,"agent_s":52814,"agent_n":26},"claude_code-glm_5_3_flash-high":{"est_usd":17.03,"est_n":46,"agent_s":116920,"agent_n":46},"claude_code-kimi_3_1_pro-high":{"est_usd":229.29,"est_n":46,"agent_s":145226,"agent_n":46},"claude_code-opus_5_5-high":{"est_usd":163.56,"est_n":46,"agent_s":82944,"agent_n":46},"codex-gpt6_1_sol-high":{"est_usd":96.54,"est_n":46,"agent_s":109918,"agent_n":46},"codex-gpt6_2_astra-high":{"est_usd":148.44,"est_n":46,"agent_s":45245,"agent_n":46},"codex-gpt6_astra-high":{"est_usd":640.64,"est_n":46,"agent_s":98834,"agent_n":46},"codex-gpt6_luna-xhigh":{"est_usd":18.51,"est_n":46,"agent_s":154382,"agent_n":46},"dsh-deepseek_v4_1_flash-high":{"est_usd":14.29,"est_n":46,"agent_s":124887,"agent_n":46}}},{"id":"ispatialgym","name":"iSpatialGym","code":"ISG","origin":"ours","example":false,"blurb":"Tests spatial reasoning from images. The goal is given only as a picture: a mobile two-armed robot rearranges the room, or moves itself, until its camera view matches it.","description":"Spatial reasoning for a mobile two-armed robot, a Galaxea R1 Pro, in BEHAVIOR-1K's houses. Each task gives only a goal image, the head camera's view once the task is done, and no sentence names the objects: line an object up with a table edge, set a tilted one upright, place it in, on, under, next to or touching another, open an appliance to the pictured angle, arrange blocks into a pattern, turn a held object to the pictured orientation, or drive and move until its own camera sees the goal view. Success is a geometric check: every measured target within tolerance, with the objects and the robot at rest over the last moments of the episode. No reference solution ships with these tasks.","simulator":"OmniGibson (BEHAVIOR-1K fork) on Isaac Sim 5.1","links":{"repository":"https://github.com/kmy17518/iSpatialGym"},"robots":[{"id":"r1pro","name":"Galaxea R1 Pro","body":"mobile"}],"body":"mobile","bodies":{"mobile":12},"instances":12,"task_dirs":24,"modes":{"privileged":12,"standard":12},"oracle":{"full":0,"partial":0,"none":12},"protocols":["eai-standard/2.2"],"budgets_min":[60],"on_site":false,"preview":[{"task":"configuration_matching_articulation_open_small_scale","title":"Configuration matching articulation open small scale","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/configuration_matching_articulation_open_small_scale.jpg?v=a595264d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/configuration_matching_articulation_open_small_scale.mp4?v=6ce18bca","motion":true,"aspect":1.778,"seconds":11},{"task":"configuration_matching_block_design","title":"Configuration matching block design","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/configuration_matching_block_design.jpg?v=068ef7b5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/configuration_matching_block_design.mp4?v=cc40d51d","motion":true,"aspect":1.778,"seconds":14},{"task":"alignment_axial","title":"Alignment axial","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/alignment_axial.jpg?v=d953c7f5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/alignment_axial.mp4?v=07ed2fee","motion":true,"aspect":1.778,"seconds":22},{"task":"alignment_vertical","title":"Alignment vertical","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/alignment_vertical.jpg?v=c7b657d1","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/alignment_vertical.mp4?v=f0aba133","motion":true,"aspect":1.778,"seconds":21},{"task":"inter_object_relation_satisfaction_inside_small_scale","title":"Inter object relation satisfaction inside small scale","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/inter_object_relation_satisfaction_inside_small_scale.jpg?v=6f9515ca","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/inter_object_relation_satisfaction_inside_small_scale.mp4?v=a84a416e","motion":true,"aspect":1.778,"seconds":15},{"task":"camera_relocalization_obstructed","title":"Camera relocalization obstructed","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/camera_relocalization_obstructed.jpg?v=7d18df73","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/camera_relocalization_obstructed.mp4?v=dc9c9f0e","motion":true,"aspect":1.778,"seconds":22},{"task":"inter_object_relation_satisfaction_nextto_small_scale","title":"Inter object relation satisfaction nextto small scale","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/inter_object_relation_satisfaction_nextto_small_scale.jpg?v=5713a8d9","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/inter_object_relation_satisfaction_nextto_small_scale.mp4?v=855732e3","motion":true,"aspect":1.778,"seconds":22},{"task":"inter_object_relation_satisfaction_ontop_small_scale","title":"Inter object relation satisfaction ontop small scale","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/inter_object_relation_satisfaction_ontop_small_scale.jpg?v=bd5dd701","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/inter_object_relation_satisfaction_ontop_small_scale.mp4?v=8173c5c5","motion":true,"aspect":1.778,"seconds":17}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/_teaser.jpg?v=a595264d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ispatialgym/_teaser.mp4?v=6ce18bca","cover":true,"aspect":1.778,"seconds":11},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":8,"success":3,"trials":8},"standard":{"done":8,"success":3,"trials":8}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":8,"success":4,"trials":8},"standard":{"done":8,"success":3,"trials":8}},"claude_code-fable_5_2-high":{"privileged":{"done":8,"success":7,"trials":8},"standard":{"done":8,"success":7,"trials":8}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":8,"success":0,"trials":8},"standard":{"done":8,"success":0,"trials":8}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":8,"success":2,"trials":8},"standard":{"done":8,"success":3,"trials":8}},"claude_code-opus_5_5-high":{"privileged":{"done":8,"success":5,"trials":8},"standard":{"done":8,"success":3,"trials":8}},"codex-gpt6_1_sol-high":{"privileged":{"done":8,"success":7,"trials":8},"standard":{"done":8,"success":5,"trials":8}},"codex-gpt6_2_astra-high":{"privileged":{"done":8,"success":8,"trials":8},"standard":{"done":8,"success":7,"trials":8}},"codex-gpt6_astra-high":{"privileged":{"done":8,"success":7,"trials":8},"standard":{"done":8,"success":4,"trials":8}},"codex-gpt6_luna-xhigh":{"privileged":{"done":8,"success":0,"trials":8},"standard":{"done":8,"success":0,"trials":8}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":8,"success":3,"trials":8},"standard":{"done":8,"success":1,"trials":8}}},"solved":{"privileged":11,"standard":10},"attempted":{"privileged":12,"standard":12},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":66.57,"est_n":16,"agent_s":42780,"agent_n":16},"antigravity_cli-gemini_4_argon-high":{"est_usd":64.14,"est_n":16,"agent_s":52658,"agent_n":16},"claude_code-fable_5_2-high":{"est_usd":104.55,"est_n":16,"agent_s":33933,"agent_n":16},"claude_code-glm_5_3_flash-high":{"est_usd":5.98,"est_n":16,"agent_s":44088,"agent_n":16},"claude_code-kimi_3_1_pro-high":{"est_usd":65.22,"est_n":16,"agent_s":45370,"agent_n":16},"claude_code-opus_5_5-high":{"est_usd":37.55,"est_n":16,"agent_s":31146,"agent_n":16},"codex-gpt6_1_sol-high":{"est_usd":30.58,"est_n":16,"agent_s":40982,"agent_n":16},"codex-gpt6_2_astra-high":{"est_usd":88.87,"est_n":16,"agent_s":29055,"agent_n":16},"codex-gpt6_astra-high":{"est_usd":225.39,"est_n":16,"agent_s":37100,"agent_n":16},"codex-gpt6_luna-xhigh":{"est_usd":4.47,"est_n":16,"agent_s":44658,"agent_n":16},"dsh-deepseek_v4_1_flash-high":{"est_usd":4.88,"est_n":16,"agent_s":41801,"agent_n":16}}},{"id":"robowits","name":"RoboWits","code":"RWT","origin":"adapted","example":false,"blurb":"Tabletop tasks that need a physical insight, like a ruler to align things or a funnel to pour.","description":"15 of its 30 seed tabletop tasks for a fixed bimanual robot, each requiring a physical insight rather than a longer motion: use a ruler to align, a dustpan to sweep, a funnel to pour, an inclined plane to move a box. 163 published mutations vary the scene while keeping the parent task's success predicate. Scored by the task's own success checker.","edition":"code release 2026-06-04 (commit 9cc30ae)","simulator":"Genesis 0.4.7 (gs_gym)","links":{"paper":"https://arxiv.org/abs/2605.30326","project_page":"https://umass-embodied-agi.github.io/RoboWits/","repository":"https://github.com/UMass-Embodied-AGI/RoboWits","dataset":"https://huggingface.co/datasets/XHRlyb2001/RoboWits_lerobot_dataset","assets":"https://huggingface.co/datasets/XHRlyb2001/RoboWits_assets"},"robots":[{"id":"marvin-bimanual","name":"Marvin","body":"bimanual"}],"body":"bimanual","bodies":{"bimanual":15},"instances":15,"task_dirs":30,"modes":{"privileged":15,"standard":15},"oracle":{"full":0,"partial":2,"none":13},"protocols":["eai-standard/2.2"],"budgets_min":[60],"tags":{"planning-reasoning":15,"control-coordination":15,"manipulation":15,"perception-understanding":2},"on_site":true,"preview":[{"task":"ball_onto_tower","title":"Ball Onto Tower","cover":true,"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/ball_onto_tower.jpg?v=9a3ab797","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/ball_onto_tower.mp4?v=0428b3f4","motion":true,"aspect":1.778,"seconds":37},{"task":"stand_pages","title":"Stand Pages","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/stand_pages.jpg?v=a6ce7df0","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/stand_pages.mp4?v=0dc51825","motion":true,"aspect":1.778,"seconds":20},{"task":"ball_into_bottle","title":"Ball Into Bottle","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/ball_into_bottle.jpg?v=77854660","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/ball_into_bottle.mp4?v=8c20b1f3","motion":true,"aspect":1.778,"seconds":17},{"task":"hold_cup","title":"Hold Cup","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/hold_cup.jpg?v=996938f4","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/hold_cup.mp4?v=60fece9d","motion":true,"aspect":1.778,"seconds":38},{"task":"raise_platform","title":"Raise Platform","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/raise_platform.jpg?v=e01c4ea2","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/raise_platform.mp4?v=eeea9ddf","motion":true,"aspect":1.778,"seconds":48},{"task":"dominos","title":"Dominos","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/dominos.jpg?v=e509c63d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/dominos.mp4?v=fa6e3f37","motion":true,"aspect":1.778,"seconds":13},{"task":"move_cube","title":"Move Cube","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/move_cube.jpg?v=6634294b","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/move_cube.mp4?v=37ab87a4","motion":true,"aspect":1.778,"seconds":19},{"task":"align_blocks","title":"Align Blocks","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/align_blocks.jpg?v=51d73128","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/align_blocks.mp4?v=51fb3b9d","motion":true,"aspect":1.778,"seconds":14}],"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":6,"success":4,"trials":6},"standard":{"done":6,"success":0,"trials":6}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":6,"success":5,"trials":6},"standard":{"done":6,"success":2,"trials":6}},"claude_code-fable_5_2-high":{"privileged":{"done":6,"success":5,"trials":6},"standard":{"done":6,"success":3,"trials":6}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":6,"success":1,"trials":6},"standard":{"done":6,"success":1,"trials":6}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":6,"success":4,"trials":6},"standard":{"done":6,"success":2,"trials":6}},"claude_code-opus_5_5-high":{"privileged":{"done":6,"success":5,"trials":6},"standard":{"done":6,"success":4,"trials":6}},"codex-gpt6_1_sol-high":{"privileged":{"done":6,"success":5,"trials":6},"standard":{"done":6,"success":3,"trials":6}},"codex-gpt6_2_astra-high":{"privileged":{"done":6,"success":6,"trials":6},"standard":{"done":6,"success":3,"trials":6}},"codex-gpt6_astra-high":{"privileged":{"done":6,"success":5,"trials":6},"standard":{"done":6,"success":2,"trials":6}},"codex-gpt6_luna-xhigh":{"privileged":{"done":6,"success":0,"trials":6},"standard":{"done":6,"success":0,"trials":6}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":6,"success":1,"trials":6},"standard":{"done":6,"success":0,"trials":6}}},"solved":{"privileged":9,"standard":11},"attempted":{"privileged":9,"standard":15},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":58.61,"est_n":12,"agent_s":34403,"agent_n":12},"antigravity_cli-gemini_4_argon-high":{"est_usd":32.92,"est_n":12,"agent_s":26414,"agent_n":12},"claude_code-fable_5_2-high":{"est_usd":108.32,"est_n":12,"agent_s":25291,"agent_n":12},"claude_code-glm_5_3_flash-high":{"est_usd":5.98,"est_n":12,"agent_s":38834,"agent_n":12},"claude_code-kimi_3_1_pro-high":{"est_usd":66.02,"est_n":12,"agent_s":37623,"agent_n":12},"claude_code-opus_5_5-high":{"est_usd":44.43,"est_n":12,"agent_s":16009,"agent_n":12},"codex-gpt6_1_sol-high":{"est_usd":16.55,"est_n":12,"agent_s":18940,"agent_n":12},"codex-gpt6_2_astra-high":{"est_usd":26.39,"est_n":12,"agent_s":7774,"agent_n":12},"codex-gpt6_astra-high":{"est_usd":186.21,"est_n":12,"agent_s":19877,"agent_n":12},"codex-gpt6_luna-xhigh":{"est_usd":5.38,"est_n":12,"agent_s":41272,"agent_n":12},"dsh-deepseek_v4_1_flash-high":{"est_usd":5.41,"est_n":12,"agent_s":40279,"agent_n":12}}},{"id":"maniguard","name":"ManiGuard","code":"MGD","origin":"adapted","example":false,"blurb":"Manipulation safety. A task counts only if no safety rule was broken on the way.","description":"A manipulation safety benchmark from the NU IDEAS Lab: tabletop tasks for a Franka Panda with long fin-ray fingers, each with a safety specification in linear temporal logic over physics predicates (dropped, upright, on top of, closed, touching), checked after every control step. A task counts only if the goal holds and no rule was ever broken.","simulator":"BEHAVIOR-1K v3.7.2 (OmniGibson 3.7.2 on Isaac Sim 4.5)","links":{"paper":"https://arxiv.org/abs/2608.17386","repository":"https://github.com/NU-IDEAS-Lab/ManiGuard","project_page":"https://nu-ideas-lab.github.io/ManiGuard/"},"robots":[{"id":"franka-panda-longfinger","name":"Franka Panda","body":"arm"}],"body":"arm","bodies":{"arm":30},"instances":30,"task_dirs":60,"modes":{"privileged":30,"standard":30},"oracle":{"full":14,"partial":0,"none":16},"protocols":["eai-standard/2.2"],"budgets_min":[60],"on_site":false,"preview":[{"task":"cabinet_pickup_i22","title":"Cabinet pickup i22","cover":true,"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/cabinet_pickup_i22.jpg?v=2e63bea5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/cabinet_pickup_i22.mp4?v=2db7ecb9","motion":true,"aspect":1.778,"seconds":41},{"task":"cabinet_pickup_i19","title":"Cabinet pickup i19","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/cabinet_pickup_i19.jpg?v=af291eb1","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/cabinet_pickup_i19.mp4?v=74c2b538","motion":true,"aspect":1.778,"seconds":39},{"task":"dusty_transfer_i24","title":"Dusty transfer i24","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/dusty_transfer_i24.jpg?v=e21cd5bb","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/dusty_transfer_i24.mp4?v=1b3ea0bd","motion":true,"aspect":1.778,"seconds":18},{"task":"lid_transport_i07","title":"Lid transport i07","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/lid_transport_i07.jpg?v=b8ca07a7","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/lid_transport_i07.mp4?v=83666f51","motion":true,"aspect":1.778,"seconds":33},{"task":"stack_retrieve_i08","title":"Stack retrieve i08","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/stack_retrieve_i08.jpg?v=4ddb6d11","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/stack_retrieve_i08.mp4?v=f7c5deb1","motion":true,"aspect":1.778,"seconds":28},{"task":"cabinet_pickup_i20","title":"Cabinet pickup i20","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/cabinet_pickup_i20.jpg?v=f9114e5f","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/cabinet_pickup_i20.mp4?v=e64527d5","motion":true,"aspect":1.778,"seconds":37},{"task":"cabinet_pickup_i26","title":"Cabinet pickup i26","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/cabinet_pickup_i26.jpg?v=3a1954e5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/cabinet_pickup_i26.mp4?v=92684cff","motion":true,"aspect":1.778,"seconds":25},{"task":"cabinet_pickup_i29","title":"Cabinet pickup i29","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/cabinet_pickup_i29.jpg?v=888bafea","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/cabinet_pickup_i29.mp4?v=bc213ebd","motion":true,"aspect":1.778,"seconds":28}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/_teaser.jpg?v=af291eb1","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/_teaser.mp4?v=74c2b538","cover":true,"aspect":1.778,"seconds":39},"results":{"antigravity_cli-gemini_3_8_flash-high":{"privileged":{"done":7,"success":3,"trials":7},"standard":{"done":7,"success":2,"trials":7}},"antigravity_cli-gemini_4_argon-high":{"privileged":{"done":7,"success":7,"trials":7},"standard":{"done":7,"success":4,"trials":7}},"claude_code-fable_5_2-high":{"privileged":{"done":7,"success":7,"trials":7},"standard":{"done":7,"success":6,"trials":7}},"claude_code-glm_5_3_flash-high":{"privileged":{"done":7,"success":0,"trials":7},"standard":{"done":7,"success":0,"trials":7}},"claude_code-kimi_3_1_pro-high":{"privileged":{"done":7,"success":5,"trials":7},"standard":{"done":7,"success":4,"trials":7}},"claude_code-opus_5_5-high":{"privileged":{"done":7,"success":7,"trials":7},"standard":{"done":7,"success":7,"trials":7}},"codex-gpt6_1_sol-high":{"privileged":{"done":7,"success":7,"trials":7},"standard":{"done":7,"success":5,"trials":7}},"codex-gpt6_2_astra-high":{"privileged":{"done":7,"success":7,"trials":7},"standard":{"done":7,"success":4,"trials":7}},"codex-gpt6_astra-high":{"privileged":{"done":7,"success":7,"trials":7},"standard":{"done":7,"success":3,"trials":7}},"codex-gpt6_luna-xhigh":{"privileged":{"done":7,"success":0,"trials":7},"standard":{"done":7,"success":0,"trials":7}},"dsh-deepseek_v4_1_flash-high":{"privileged":{"done":7,"success":2,"trials":7},"standard":{"done":7,"success":0,"trials":7}}},"solved":{"privileged":27,"standard":19},"attempted":{"privileged":30,"standard":30},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":56.01,"est_n":14,"agent_s":39411,"agent_n":14},"antigravity_cli-gemini_4_argon-high":{"est_usd":44.2,"est_n":14,"agent_s":38815,"agent_n":14},"claude_code-fable_5_2-high":{"est_usd":61.7,"est_n":14,"agent_s":18592,"agent_n":14},"claude_code-glm_5_3_flash-high":{"est_usd":8.08,"est_n":14,"agent_s":75097,"agent_n":14},"claude_code-kimi_3_1_pro-high":{"est_usd":58.25,"est_n":14,"agent_s":40704,"agent_n":14},"claude_code-opus_5_5-high":{"est_usd":36.22,"est_n":14,"agent_s":26106,"agent_n":14},"codex-gpt6_1_sol-high":{"est_usd":19.51,"est_n":14,"agent_s":24256,"agent_n":14},"codex-gpt6_2_astra-high":{"est_usd":43.36,"est_n":14,"agent_s":17527,"agent_n":14},"codex-gpt6_astra-high":{"est_usd":176.66,"est_n":14,"agent_s":33379,"agent_n":14},"codex-gpt6_luna-xhigh":{"est_usd":5.12,"est_n":14,"agent_s":50158,"agent_n":14},"dsh-deepseek_v4_1_flash-high":{"est_usd":6.09,"est_n":14,"agent_s":58486,"agent_n":14}}},{"id":"roboengineer","name":"RoboEngineer","code":"ENG","origin":"ours","example":false,"blurb":"Tests the sim-to-real engineering loop. The agent builds a controller in a public simulator, deploys it to one with hidden dynamics, and fixes it from what the deployment shows.","description":"The loop of real sim-to-real engineering, in simulation. The agent tunes a controller or trains a policy in a public simulator, deploys it to a private evaluator whose dynamics differ in hidden ways, and revises it from the noisy telemetry and camera frames that come back, over at most three deployments. The robots are quadrotors (landing on a pad, flying figure eights in wind), a racing car (tracking, drifting into a parking bay, timed laps), a Unitree Go2 (body attitude, jumps, weaving between poles) and a Unitree G1 that learns to walk over rough and stepped ground on the agent's own GPU. The chosen policy passes only if it succeeds on every episode of a held-out set on the same hidden dynamics, so tuning to the deployments it has seen is not enough.","simulator":"MuJoCo and learned dynamics surrogates (quadrotors, a racing car, Unitree Go2 and G1)","robots":[{"id":"anycar","name":"AnyCar racing car","body":"mobile"},{"id":"intel-aero-partial","name":"Intel Aero quadrotor","body":"aerial"},{"id":"neural-fly-partial","name":"Neural-Fly quadrotor","body":"aerial"},{"id":"unitree-g1","name":"Unitree G1","body":"humanoid"},{"id":"unitree-go2","name":"Unitree Go2","body":"quadruped"}],"body":"mobile","embodiment_label":"Diverse agile mobile robots","bodies":{"mobile":10,"humanoid":2,"quadruped":8,"aerial":10},"instances":30,"task_dirs":30,"modes":{"privileged":0,"standard":30},"oracle":{"full":16,"partial":5,"none":9},"protocols":["eai-policy/1.0","eai-policy/1.1"],"budgets_min":[60,240],"on_site":false,"preview":[{"task":"anycar_compact_lap_monaco_6","title":"Anycar compact lap monaco 6","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/anycar_compact_lap_monaco_6.jpg?v=36e10c5e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/anycar_compact_lap_monaco_6.mp4?v=794f0800","motion":true,"aspect":1.778,"seconds":6},{"task":"go2_weave_i01","title":"Go2 weave i01","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/go2_weave_i01.jpg?v=e7f27423","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/go2_weave_i01.mp4?v=be67934b","motion":true,"aspect":1.778,"seconds":20},{"task":"go2_yaw_jump","title":"Go2 yaw jump","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/go2_yaw_jump.jpg?v=d18a5571","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/go2_yaw_jump.mp4?v=840d65fa","motion":true,"aspect":1.778,"seconds":6},{"task":"go2_yaw_jump_i01","title":"Go2 yaw jump i01","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/go2_yaw_jump_i01.jpg?v=10bc1d48","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/go2_yaw_jump_i01.mp4?v=31d2b58f","motion":true,"aspect":1.778,"seconds":6},{"task":"landing_random_pad","title":"Landing random pad","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/landing_random_pad.jpg?v=10610e41","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/landing_random_pad.mp4?v=a4c643f2","motion":true,"aspect":1.778,"seconds":16},{"task":"neuralfly_eight","title":"Neuralfly eight","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/neuralfly_eight.jpg?v=9627ff7a","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/neuralfly_eight.mp4?v=7d698d3d","motion":true,"aspect":1.778,"seconds":38},{"task":"anycar_compact_lap_monaco_6_i01","title":"Anycar compact lap monaco 6 i01","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/anycar_compact_lap_monaco_6_i01.jpg?v=42e78273","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/anycar_compact_lap_monaco_6_i01.mp4?v=4fd6dc5d","motion":true,"aspect":1.778,"seconds":6},{"task":"anycar_compact_lap_silverstone_8","title":"Anycar compact lap silverstone 8","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/anycar_compact_lap_silverstone_8.jpg?v=f819c953","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/anycar_compact_lap_silverstone_8.mp4?v=639a7421","motion":true,"aspect":1.778,"seconds":6}],"teaser":{"poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/_teaser.jpg?v=18abfbfc","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboengineer/_teaser.mp4?v=6a239ea8","cover":true,"aspect":1.778,"seconds":2},"results":{"antigravity_cli-gemini_3_8_flash-high":{"standard":{"done":15,"success":0,"trials":15}},"antigravity_cli-gemini_4_argon-high":{"standard":{"done":15,"success":4,"trials":15}},"claude_code-fable_5_2-high":{"standard":{"done":15,"success":7,"trials":15}},"claude_code-glm_5_3_flash-high":{"standard":{"done":15,"success":2,"trials":15}},"claude_code-kimi_3_1_pro-high":{"standard":{"done":15,"success":2,"trials":15}},"claude_code-opus_5_5-high":{"standard":{"done":15,"success":4,"trials":15}},"codex-gpt6_1_sol-high":{"standard":{"done":15,"success":3,"trials":15}},"codex-gpt6_2_astra-high":{"standard":{"done":15,"success":3,"trials":15}},"codex-gpt6_astra-high":{"standard":{"done":15,"success":5,"trials":15}},"codex-gpt6_luna-xhigh":{"standard":{"done":15,"success":0,"trials":15}},"dsh-deepseek_v4_1_flash-high":{"standard":{"done":15,"success":2,"trials":15}}},"solved":{"privileged":0,"standard":25},"attempted":{"privileged":0,"standard":30},"cost":{"antigravity_cli-gemini_3_8_flash-high":{"est_usd":48.8,"est_n":15,"agent_s":34799,"agent_n":15},"antigravity_cli-gemini_4_argon-high":{"est_usd":170.79,"est_n":15,"agent_s":147001,"agent_n":15},"claude_code-fable_5_2-high":{"est_usd":283.67,"est_n":15,"agent_s":129113,"agent_n":15},"claude_code-glm_5_3_flash-high":{"est_usd":8.27,"est_n":15,"agent_s":110327,"agent_n":15},"claude_code-kimi_3_1_pro-high":{"est_usd":241.45,"est_n":15,"agent_s":106394,"agent_n":15},"claude_code-opus_5_5-high":{"est_usd":73.86,"est_n":15,"agent_s":136869,"agent_n":15},"codex-gpt6_1_sol-high":{"est_usd":126.11,"est_n":15,"agent_s":141711,"agent_n":15},"codex-gpt6_2_astra-high":{"est_usd":373.98,"est_n":15,"agent_s":116984,"agent_n":15},"codex-gpt6_astra-high":{"est_usd":654.4,"est_n":15,"agent_s":115873,"agent_n":15},"codex-gpt6_luna-xhigh":{"est_usd":2.16,"est_n":15,"agent_s":26411,"agent_n":15},"dsh-deepseek_v4_1_flash-high":{"est_usd":6.57,"est_n":15,"agent_s":96493,"agent_n":15}}}],"bodies":{"arm":"One arm","bimanual":"Two arms","mobile":"Mobile robot","humanoid":"Humanoid","quadruped":"Quadruped","hand":"Dexterous hand","musculoskeletal":"Musculoskeletal","aerial":"Aerial robot"},"taxonomy":[{"id":"capability","name":"Capability","description":"What the agent has to be able to do to solve the task. Take every one the goal requires.","tags":[{"id":"perception-understanding","name":"Perception & Understanding","description":"Perception, object/state recognition, semantic grounding, spatial understanding."},{"id":"planning-reasoning","name":"Planning & Reasoning","description":"Task planning, sequencing, tool/affordance reasoning, geometric reasoning."},{"id":"control-coordination","name":"Control & Coordination","description":"Manipulation/control, contact-rich execution, locomotion control, multi-arm or whole-body coordination."},{"id":"feedback-adaptation","name":"Feedback & Adaptation","description":"Closed-loop correction, failure recovery, online adaptation, exploration based on feedback."}]},{"id":"domain","name":"Task Domain","description":"What kind of task it is: where the robot works and what its body has to do. Take every one that applies.","tags":[{"id":"manipulation","name":"Manipulation","description":"Fixed-base object manipulation, grasping, pushing, insertion, tool use, articulated/deformable object interaction."},{"id":"locomotion-stability","name":"Locomotion & Stability","description":"Walking, running, balancing, terrain traversal, dynamic stabilization."},{"id":"navigation-exploration","name":"Navigation & Exploration","description":"Navigation, target search, exploration, spatial memory."},{"id":"mobile-whole-body-manipulation","name":"Mobile / Whole-body Manipulation","description":"Tasks combining navigation/locomotion with manipulation or requiring coordinated whole-body motion."},{"id":"interaction-collaboration","name":"Interaction & Collaboration","description":"Human-robot interaction, multi-robot collaboration, handover, cooperative tasks."}]}],"runs":[{"id":"antigravity_cli-gemini_3_8_flash-high","model":"Gemini 3.8 Flash","harness":"Antigravity","effort":"high","label":"Antigravity + Gemini 3.8 Flash, reasoning high","kind":"agent","done":{"privileged":416,"standard":535},"success":{"privileged":102,"standard":33},"suites":["behavior-1k","robolab","robocasa","robocasa365","robocasa-gr1","metaworldplus","vlabench","humanoidbench","molmospaces","kinder","robopaint","robotwin-2","mujoco-playground","dextoolbench","dexjoco","myosuite","roboarc","robodojo","pandadyn","bigym2","yamlab","libero-max","embodied-olympics-fun","embodied-olympics-match","ambench","embodiedswe","embodied-safety-exam","dexhand","ispatialgym","robowits","maniguard","roboengineer"],"est_usd":6657.08,"est_n":946,"agent_s":4398506,"agent_n":951},{"id":"antigravity_cli-gemini_4_argon-high","model":"Gemini 4 Argon","harness":"Antigravity","effort":"high","label":"Antigravity + Gemini 4 Argon, reasoning high","kind":"agent","done":{"privileged":416,"standard":535},"success":{"privileged":188,"standard":132},"suites":["behavior-1k","robolab","robocasa","robocasa365","robocasa-gr1","metaworldplus","vlabench","humanoidbench","molmospaces","kinder","robopaint","robotwin-2","mujoco-playground","dextoolbench","dexjoco","myosuite","roboarc","robodojo","pandadyn","bigym2","yamlab","libero-max","embodied-olympics-fun","embodied-olympics-match","ambench","embodiedswe","embodied-safety-exam","dexhand","ispatialgym","robowits","maniguard","roboengineer"],"est_usd":4894.58,"est_n":946,"agent_s":4182551,"agent_n":951},{"id":"claude_code-fable_5_2-high","model":"Claude Fable 5.2","harness":"Claude Code","effort":"high","label":"Claude Code + Claude Fable 5.2, reasoning high","kind":"agent","done":{"privileged":416,"standard":535},"success":{"privileged":337,"standard":270},"suites":["behavior-1k","robolab","robocasa","robocasa365","robocasa-gr1","metaworldplus","vlabench","humanoidbench","molmospaces","kinder","robopaint","robotwin-2","mujoco-playground","dextoolbench","dexjoco","myosuite","roboarc","robodojo","pandadyn","bigym2","yamlab","libero-max","embodied-olympics-fun","embodied-olympics-match","ambench","embodiedswe","embodied-safety-exam","dexhand","ispatialgym","robowits","maniguard","roboengineer"],"est_usd":11461.87,"est_n":944,"agent_s":2757036,"agent_n":808},{"id":"claude_code-glm_5_3_flash-high","model":"GLM 5.3 Flash","harness":"Claude Code","effort":"high","label":"Claude Code + GLM 5.3 Flash, reasoning high","kind":"agent","done":{"privileged":416,"standard":535},"success":{"privileged":44,"standard":12},"suites":["behavior-1k","robolab","robocasa","robocasa365","robocasa-gr1","metaworldplus","vlabench","humanoidbench","molmospaces","kinder","robopaint","robotwin-2","mujoco-playground","dextoolbench","dexjoco","myosuite","roboarc","robodojo","pandadyn","bigym2","yamlab","libero-max","embodied-olympics-fun","embodied-olympics-match","ambench","embodiedswe","embodied-safety-exam","dexhand","ispatialgym","robowits","maniguard","roboengineer"],"est_usd":574.18,"est_n":946,"agent_s":4911436,"agent_n":951},{"id":"claude_code-kimi_3_1_pro-high","model":"Kimi 3.1 Pro","harness":"Claude Code","effort":"high","label":"Claude Code + Kimi 3.1 Pro, reasoning high","kind":"agent","done":{"privileged":416,"standard":535},"success":{"privileged":165,"standard":59},"suites":["behavior-1k","robolab","robocasa","robocasa365","robocasa-gr1","metaworldplus","vlabench","humanoidbench","molmospaces","kinder","robopaint","robotwin-2","mujoco-playground","dextoolbench","dexjoco","myosuite","roboarc","robodojo","pandadyn","bigym2","yamlab","libero-max","embodied-olympics-fun","embodied-olympics-match","ambench","embodiedswe","embodied-safety-exam","dexhand","ispatialgym","robowits","maniguard","roboengineer"],"est_usd":7365.78,"est_n":946,"agent_s":4516141,"agent_n":951},{"id":"claude_code-opus_5_5-high","model":"Claude Opus 5.5","harness":"Claude Code","effort":"high","label":"Claude Code + Claude Opus 5.5, reasoning high","kind":"agent","done":{"privileged":416,"standard":535},"success":{"privileged":257,"standard":134},"suites":["behavior-1k","robolab","robocasa","robocasa365","robocasa-gr1","metaworldplus","vlabench","humanoidbench","molmospaces","kinder","robopaint","robotwin-2","mujoco-playground","dextoolbench","dexjoco","myosuite","roboarc","robodojo","pandadyn","bigym2","yamlab","libero-max","embodied-olympics-fun","embodied-olympics-match","ambench","embodiedswe","embodied-safety-exam","dexhand","ispatialgym","robowits","maniguard","roboengineer"],"est_usd":5080.02,"est_n":924,"agent_s":2873318,"agent_n":929},{"id":"codex-gpt6_1_sol-high","model":"GPT-6.1 Sol","harness":"Codex","effort":"high","label":"Codex + GPT-6.1 Sol, reasoning high","kind":"agent","done":{"privileged":416,"standard":535},"success":{"privileged":309,"standard":176},"suites":["behavior-1k","robolab","robocasa","robocasa365","robocasa-gr1","metaworldplus","vlabench","humanoidbench","molmospaces","kinder","robopaint","robotwin-2","mujoco-playground","dextoolbench","dexjoco","myosuite","roboarc","robodojo","pandadyn","bigym2","yamlab","libero-max","embodied-olympics-fun","embodied-olympics-match","ambench","embodiedswe","embodied-safety-exam","dexhand","ispatialgym","robowits","maniguard","roboengineer"],"est_usd":3607.57,"est_n":946,"agent_s":3962393,"agent_n":951},{"id":"codex-gpt6_2_astra-high","model":"GPT-6.2 Astra","harness":"Codex","effort":"high","label":"Codex + GPT-6.2 Astra, reasoning high","kind":"agent","done":{"privileged":416,"standard":535},"success":{"privileged":395,"standard":301},"suites":["behavior-1k","robolab","robocasa","robocasa365","robocasa-gr1","metaworldplus","vlabench","humanoidbench","molmospaces","kinder","robopaint","robotwin-2","mujoco-playground","dextoolbench","dexjoco","myosuite","roboarc","robodojo","pandadyn","bigym2","yamlab","libero-max","embodied-olympics-fun","embodied-olympics-match","ambench","embodiedswe","embodied-safety-exam","dexhand","ispatialgym","robowits","maniguard","roboengineer"],"est_usd":6573.12,"est_n":951,"agent_s":2406689,"agent_n":951},{"id":"codex-gpt6_astra-high","model":"GPT-6 Astra","harness":"Codex","effort":"high","label":"Codex + GPT-6 Astra, reasoning high","kind":"agent","done":{"privileged":416,"standard":535},"success":{"privileged":333,"standard":192},"suites":["behavior-1k","robolab","robocasa","robocasa365","robocasa-gr1","metaworldplus","vlabench","humanoidbench","molmospaces","kinder","robopaint","robotwin-2","mujoco-playground","dextoolbench","dexjoco","myosuite","roboarc","robodojo","pandadyn","bigym2","yamlab","libero-max","embodied-olympics-fun","embodied-olympics-match","ambench","embodiedswe","embodied-safety-exam","dexhand","ispatialgym","robowits","maniguard","roboengineer"],"est_usd":22498.78,"est_n":945,"agent_s":3942580,"agent_n":950},{"id":"codex-gpt6_luna-xhigh","model":"GPT-6 Luna","harness":"Codex","effort":"xhigh","label":"Codex + GPT-6 Luna, reasoning xhigh","kind":"agent","done":{"privileged":416,"standard":535},"success":{"privileged":45,"standard":6},"suites":["behavior-1k","robolab","robocasa","robocasa365","robocasa-gr1","metaworldplus","vlabench","humanoidbench","molmospaces","kinder","robopaint","robotwin-2","mujoco-playground","dextoolbench","dexjoco","myosuite","roboarc","robodojo","pandadyn","bigym2","yamlab","libero-max","embodied-olympics-fun","embodied-olympics-match","ambench","embodiedswe","embodied-safety-exam","dexhand","ispatialgym","robowits","maniguard","roboengineer"],"est_usd":519.02,"est_n":951,"agent_s":5008167,"agent_n":951},{"id":"dsh-deepseek_v4_1_flash-high","model":"DeepSeek V4.1 Flash","harness":"DeepSeek Harness","effort":"high","label":"DeepSeek Harness + DeepSeek V4.1 Flash, reasoning high","kind":"agent","done":{"privileged":416,"standard":535},"success":{"privileged":73,"standard":18},"suites":["behavior-1k","robolab","robocasa","robocasa365","robocasa-gr1","metaworldplus","vlabench","humanoidbench","molmospaces","kinder","robopaint","robotwin-2","mujoco-playground","dextoolbench","dexjoco","myosuite","roboarc","robodojo","pandadyn","bigym2","yamlab","libero-max","embodied-olympics-fun","embodied-olympics-match","ambench","embodiedswe","embodied-safety-exam","dexhand","ispatialgym","robowits","maniguard","roboengineer"],"est_usd":390.19,"est_n":946,"agent_s":3791753,"agent_n":951}],"leaderboard":{"min_suites":10,"note":"Each view counts the core tests of one setting, and a task can have a test in more than one setting, so their counts add up to more than 481. Results of the batch evaluation from 2 October to 20 November 2026: the latest finished attempt per agent and test.","notes":[{"icon":"shield","title":"Solved means marked 1","visual":"states","text":"The verifier marked the answer 1, on a fresh replay (privileged) or the live episode (standard), within the token budget."},{"icon":"pair","title":"One setting at a time","visual":"modes","text":"Privileged is the open-book mode, standard the closed-book one, realtime the closed-book one on a clock that never stops. Each is scored and ranked apart."},{"icon":"venn","title":"Each model on its own tests","visual":"venn","text":"a / b reads solved / tested for each model."},{"icon":"podium","title":"Ranked from {min_suites} suites","text":"A model is ranked once it has marked answers in {min_suites} suites. Smaller runs are listed under the board."},{"icon":"clock","title":"The same budget","visual":"budget","text":"One attempt per test, under the same limits for every model."},{"icon":"target","title":"The core set","text":"Counts the core set of the exam's repository at {commit}. Experts among the authors and a coding agent selected it through trial runs, choosing tasks that are challenging and representative of their suite."}],"shared":{"runs":["antigravity_cli-gemini_3_8_flash-high","antigravity_cli-gemini_4_argon-high","claude_code-fable_5_2-high","claude_code-glm_5_3_flash-high","claude_code-kimi_3_1_pro-high","claude_code-opus_5_5-high","codex-gpt6_1_sol-high","codex-gpt6_2_astra-high","codex-gpt6_astra-high","dsh-deepseek_v4_1_flash-high"],"views":{"overall":{"tests":481,"scores":{"antigravity_cli-gemini_3_8_flash-high":[25,481,481],"antigravity_cli-gemini_4_argon-high":[97,481,481],"claude_code-fable_5_2-high":[225,481,481],"claude_code-glm_5_3_flash-high":[11,481,481],"claude_code-kimi_3_1_pro-high":[44,481,481],"claude_code-opus_5_5-high":[117,481,481],"codex-gpt6_1_sol-high":[151,481,481],"codex-gpt6_2_astra-high":[260,481,481],"codex-gpt6_astra-high":[162,481,481],"dsh-deepseek_v4_1_flash-high":[14,481,481]},"suites":{"antigravity_cli-gemini_3_8_flash-high":{"behavior-1k":[0,71],"robolab":[1,6],"robocasa":[0,2],"robocasa365":[0,5],"robocasa-gr1":[0,1],"metaworldplus":[1,2],"vlabench":[0,2],"humanoidbench":[0,8],"molmospaces":[0,2],"kinder":[0,4],"robopaint":[0,22],"robotwin-2":[0,3],"mujoco-playground":[0,7],"dextoolbench":[0,20],"dexjoco":[1,2],"myosuite":[0,13],"roboarc":[2,58],"robodojo":[2,37],"pandadyn":[0,10],"bigym2":[0,10],"yamlab":[0,2],"libero-max":[1,2],"embodied-olympics-fun":[1,56],"embodied-olympics-match":[0,13],"ambench":[2,2],"embodiedswe":[2,12],"embodied-safety-exam":[10,50],"dexhand":[1,23],"ispatialgym":[1,8],"robowits":[0,6],"maniguard":[0,7],"roboengineer":[0,15]},"antigravity_cli-gemini_4_argon-high":{"behavior-1k":[3,71],"robolab":[0,6],"robocasa":[1,2],"robocasa365":[0,5],"robocasa-gr1":[0,1],"metaworldplus":[2,2],"vlabench":[0,2],"humanoidbench":[0,8],"molmospaces":[2,2],"kinder":[0,4],"robopaint":[1,22],"robotwin-2":[1,3],"mujoco-playground":[2,7],"dextoolbench":[0,20],"dexjoco":[1,2],"myosuite":[0,13],"roboarc":[6,58],"robodojo":[17,37],"pandadyn":[0,10],"bigym2":[0,10],"yamlab":[1,2],"libero-max":[2,2],"embodied-olympics-fun":[8,56],"embodied-olympics-match":[3,13],"ambench":[0,2],"embodiedswe":[4,12],"embodied-safety-exam":[27,50],"dexhand":[4,23],"ispatialgym":[2,8],"robowits":[2,6],"maniguard":[4,7],"roboengineer":[4,15]},"claude_code-fable_5_2-high":{"behavior-1k":[19,71],"robolab":[2,6],"robocasa":[2,2],"robocasa365":[2,5],"robocasa-gr1":[1,1],"metaworldplus":[2,2],"vlabench":[1,2],"humanoidbench":[0,8],"molmospaces":[2,2],"kinder":[1,4],"robopaint":[2,22],"robotwin-2":[1,3],"mujoco-playground":[3,7],"dextoolbench":[1,20],"dexjoco":[2,2],"myosuite":[5,13],"roboarc":[33,58],"robodojo":[28,37],"pandadyn":[3,10],"bigym2":[1,10],"yamlab":[2,2],"libero-max":[2,2],"embodied-olympics-fun":[25,56],"embodied-olympics-match":[5,13],"ambench":[2,2],"embodiedswe":[8,12],"embodied-safety-exam":[34,50],"dexhand":[15,23],"ispatialgym":[6,8],"robowits":[2,6],"maniguard":[6,7],"roboengineer":[7,15]},"claude_code-glm_5_3_flash-high":{"behavior-1k":[0,71],"robolab":[0,6],"robocasa":[0,2],"robocasa365":[0,5],"robocasa-gr1":[0,1],"metaworldplus":[1,2],"vlabench":[0,2],"humanoidbench":[0,8],"molmospaces":[1,2],"kinder":[0,4],"robopaint":[0,22],"robotwin-2":[0,3],"mujoco-playground":[0,7],"dextoolbench":[0,20],"dexjoco":[0,2],"myosuite":[0,13],"roboarc":[0,58],"robodojo":[1,37],"pandadyn":[0,10],"bigym2":[0,10],"yamlab":[0,2],"libero-max":[0,2],"embodied-olympics-fun":[0,56],"embodied-olympics-match":[1,13],"ambench":[0,2],"embodiedswe":[0,12],"embodied-safety-exam":[5,50],"dexhand":[0,23],"ispatialgym":[0,8],"robowits":[0,6],"maniguard":[0,7],"roboengineer":[2,15]},"claude_code-kimi_3_1_pro-high":{"behavior-1k":[0,71],"robolab":[0,6],"robocasa":[2,2],"robocasa365":[0,5],"robocasa-gr1":[0,1],"metaworldplus":[2,2],"vlabench":[0,2],"humanoidbench":[0,8],"molmospaces":[1,2],"kinder":[0,4],"robopaint":[0,22],"robotwin-2":[0,3],"mujoco-playground":[0,7],"dextoolbench":[0,20],"dexjoco":[0,2],"myosuite":[0,13],"roboarc":[0,58],"robodojo":[9,37],"pandadyn":[0,10],"bigym2":[0,10],"yamlab":[0,2],"libero-max":[1,2],"embodied-olympics-fun":[6,56],"embodied-olympics-match":[2,13],"ambench":[2,2],"embodiedswe":[0,12],"embodied-safety-exam":[9,50],"dexhand":[2,23],"ispatialgym":[1,8],"robowits":[2,6],"maniguard":[3,7],"roboengineer":[2,15]},"claude_code-opus_5_5-high":{"behavior-1k":[4,71],"robolab":[1,6],"robocasa":[2,2],"robocasa365":[0,5],"robocasa-gr1":[1,1],"metaworldplus":[2,2],"vlabench":[1,2],"humanoidbench":[0,8],"molmospaces":[2,2],"kinder":[0,4],"robopaint":[1,22],"robotwin-2":[3,3],"mujoco-playground":[0,7],"dextoolbench":[0,20],"dexjoco":[2,2],"myosuite":[3,13],"roboarc":[15,58],"robodojo":[18,37],"pandadyn":[1,10],"bigym2":[0,10],"yamlab":[2,2],"libero-max":[2,2],"embodied-olympics-fun":[13,56],"embodied-olympics-match":[0,13],"ambench":[1,2],"embodiedswe":[5,12],"embodied-safety-exam":[19,50],"dexhand":[2,23],"ispatialgym":[3,8],"robowits":[3,6],"maniguard":[7,7],"roboengineer":[4,15]},"codex-gpt6_1_sol-high":{"behavior-1k":[10,71],"robolab":[2,6],"robocasa":[2,2],"robocasa365":[1,5],"robocasa-gr1":[0,1],"metaworldplus":[2,2],"vlabench":[1,2],"humanoidbench":[0,8],"molmospaces":[2,2],"kinder":[1,4],"robopaint":[6,22],"robotwin-2":[2,3],"mujoco-playground":[3,7],"dextoolbench":[0,20],"dexjoco":[0,2],"myosuite":[3,13],"roboarc":[14,58],"robodojo":[16,37],"pandadyn":[0,10],"bigym2":[0,10],"yamlab":[1,2],"libero-max":[2,2],"embodied-olympics-fun":[19,56],"embodied-olympics-match":[4,13],"ambench":[2,2],"embodiedswe":[5,12],"embodied-safety-exam":[29,50],"dexhand":[9,23],"ispatialgym":[4,8],"robowits":[3,6],"maniguard":[5,7],"roboengineer":[3,15]},"codex-gpt6_2_astra-high":{"behavior-1k":[38,71],"robolab":[2,6],"robocasa":[2,2],"robocasa365":[3,5],"robocasa-gr1":[1,1],"metaworldplus":[2,2],"vlabench":[1,2],"humanoidbench":[0,8],"molmospaces":[2,2],"kinder":[2,4],"robopaint":[21,22],"robotwin-2":[3,3],"mujoco-playground":[7,7],"dextoolbench":[3,20],"dexjoco":[2,2],"myosuite":[7,13],"roboarc":[32,58],"robodojo":[22,37],"pandadyn":[3,10],"bigym2":[1,10],"yamlab":[2,2],"libero-max":[2,2],"embodied-olympics-fun":[23,56],"embodied-olympics-match":[7,13],"ambench":[1,2],"embodiedswe":[8,12],"embodied-safety-exam":[26,50],"dexhand":[20,23],"ispatialgym":[7,8],"robowits":[3,6],"maniguard":[4,7],"roboengineer":[3,15]},"codex-gpt6_astra-high":{"behavior-1k":[11,71],"robolab":[0,6],"robocasa":[0,2],"robocasa365":[1,5],"robocasa-gr1":[0,1],"metaworldplus":[1,2],"vlabench":[0,2],"humanoidbench":[1,8],"molmospaces":[2,2],"kinder":[0,4],"robopaint":[7,22],"robotwin-2":[1,3],"mujoco-playground":[4,7],"dextoolbench":[2,20],"dexjoco":[1,2],"myosuite":[3,13],"roboarc":[14,58],"robodojo":[21,37],"pandadyn":[1,10],"bigym2":[1,10],"yamlab":[1,2],"libero-max":[2,2],"embodied-olympics-fun":[20,56],"embodied-olympics-match":[3,13],"ambench":[2,2],"embodiedswe":[5,12],"embodied-safety-exam":[31,50],"dexhand":[13,23],"ispatialgym":[4,8],"robowits":[2,6],"maniguard":[3,7],"roboengineer":[5,15]},"dsh-deepseek_v4_1_flash-high":{"behavior-1k":[0,71],"robolab":[0,6],"robocasa":[1,2],"robocasa365":[0,5],"robocasa-gr1":[0,1],"metaworldplus":[1,2],"vlabench":[0,2],"humanoidbench":[0,8],"molmospaces":[1,2],"kinder":[0,4],"robopaint":[0,22],"robotwin-2":[0,3],"mujoco-playground":[0,7],"dextoolbench":[0,20],"dexjoco":[0,2],"myosuite":[0,13],"roboarc":[0,58],"robodojo":[3,37],"pandadyn":[0,10],"bigym2":[0,10],"yamlab":[0,2],"libero-max":[0,2],"embodied-olympics-fun":[2,56],"embodied-olympics-match":[0,13],"ambench":[2,2],"embodiedswe":[0,12],"embodied-safety-exam":[2,50],"dexhand":[0,23],"ispatialgym":[0,8],"robowits":[0,6],"maniguard":[0,7],"roboengineer":[2,15]}}},"standard":{"tests":481,"scores":{"antigravity_cli-gemini_3_8_flash-high":[32,481,481],"antigravity_cli-gemini_4_argon-high":[127,481,481],"claude_code-fable_5_2-high":[248,481,481],"claude_code-glm_5_3_flash-high":[12,481,481],"claude_code-kimi_3_1_pro-high":[55,481,481],"claude_code-opus_5_5-high":[130,481,481],"codex-gpt6_1_sol-high":[163,481,481],"codex-gpt6_2_astra-high":[279,481,481],"codex-gpt6_astra-high":[175,481,481],"dsh-deepseek_v4_1_flash-high":[17,481,481]},"suites":{"antigravity_cli-gemini_3_8_flash-high":{"behavior-1k":[0,71],"robolab":[1,6],"robocasa":[0,2],"robocasa365":[0,5],"robocasa-gr1":[0,1],"metaworldplus":[1,2],"vlabench":[0,2],"humanoidbench":[0,8],"molmospaces":[0,2],"kinder":[0,4],"robopaint":[0,22],"robotwin-2":[0,3],"mujoco-playground":[0,7],"dextoolbench":[0,20],"dexjoco":[1,2],"myosuite":[0,13],"roboarc":[2,58],"robodojo":[3,37],"pandadyn":[0,10],"bigym2":[0,10],"yamlab":[1,2],"libero-max":[1,2],"embodied-olympics-fun":[2,56],"embodied-olympics-match":[0,13],"ambench":[2,2],"embodiedswe":[2,12],"embodied-safety-exam":[10,50],"dexhand":[1,23],"ispatialgym":[3,8],"robowits":[0,6],"maniguard":[2,7],"roboengineer":[0,15]},"antigravity_cli-gemini_4_argon-high":{"behavior-1k":[4,71],"robolab":[1,6],"robocasa":[1,2],"robocasa365":[0,5],"robocasa-gr1":[0,1],"metaworldplus":[2,2],"vlabench":[0,2],"humanoidbench":[0,8],"molmospaces":[2,2],"kinder":[0,4],"robopaint":[2,22],"robotwin-2":[1,3],"mujoco-playground":[2,7],"dextoolbench":[0,20],"dexjoco":[1,2],"myosuite":[1,13],"roboarc":[11,58],"robodojo":[21,37],"pandadyn":[2,10],"bigym2":[2,10],"yamlab":[1,2],"libero-max":[2,2],"embodied-olympics-fun":[13,56],"embodied-olympics-match":[6,13],"ambench":[2,2],"embodiedswe":[4,12],"embodied-safety-exam":[27,50],"dexhand":[6,23],"ispatialgym":[3,8],"robowits":[2,6],"maniguard":[4,7],"roboengineer":[4,15]},"claude_code-fable_5_2-high":{"behavior-1k":[26,71],"robolab":[2,6],"robocasa":[2,2],"robocasa365":[2,5],"robocasa-gr1":[1,1],"metaworldplus":[2,2],"vlabench":[1,2],"humanoidbench":[0,8],"molmospaces":[2,2],"kinder":[1,4],"robopaint":[2,22],"robotwin-2":[1,3],"mujoco-playground":[3,7],"dextoolbench":[1,20],"dexjoco":[2,2],"myosuite":[5,13],"roboarc":[33,58],"robodojo":[28,37],"pandadyn":[4,10],"bigym2":[6,10],"yamlab":[2,2],"libero-max":[2,2],"embodied-olympics-fun":[27,56],"embodied-olympics-match":[10,13],"ambench":[2,2],"embodiedswe":[8,12],"embodied-safety-exam":[35,50],"dexhand":[15,23],"ispatialgym":[7,8],"robowits":[3,6],"maniguard":[6,7],"roboengineer":[7,15]},"claude_code-glm_5_3_flash-high":{"behavior-1k":[0,71],"robolab":[0,6],"robocasa":[0,2],"robocasa365":[0,5],"robocasa-gr1":[0,1],"metaworldplus":[1,2],"vlabench":[0,2],"humanoidbench":[0,8],"molmospaces":[1,2],"kinder":[0,4],"robopaint":[0,22],"robotwin-2":[0,3],"mujoco-playground":[0,7],"dextoolbench":[0,20],"dexjoco":[0,2],"myosuite":[0,13],"roboarc":[0,58],"robodojo":[1,37],"pandadyn":[0,10],"bigym2":[0,10],"yamlab":[0,2],"libero-max":[0,2],"embodied-olympics-fun":[0,56],"embodied-olympics-match":[1,13],"ambench":[0,2],"embodiedswe":[0,12],"embodied-safety-exam":[5,50],"dexhand":[0,23],"ispatialgym":[0,8],"robowits":[1,6],"maniguard":[0,7],"roboengineer":[2,15]},"claude_code-kimi_3_1_pro-high":{"behavior-1k":[0,71],"robolab":[0,6],"robocasa":[2,2],"robocasa365":[0,5],"robocasa-gr1":[1,1],"metaworldplus":[2,2],"vlabench":[0,2],"humanoidbench":[0,8],"molmospaces":[1,2],"kinder":[0,4],"robopaint":[0,22],"robotwin-2":[1,3],"mujoco-playground":[0,7],"dextoolbench":[0,20],"dexjoco":[0,2],"myosuite":[1,13],"roboarc":[1,58],"robodojo":[10,37],"pandadyn":[0,10],"bigym2":[0,10],"yamlab":[0,2],"libero-max":[1,2],"embodied-olympics-fun":[7,56],"embodied-olympics-match":[3,13],"ambench":[2,2],"embodiedswe":[0,12],"embodied-safety-exam":[10,50],"dexhand":[2,23],"ispatialgym":[3,8],"robowits":[2,6],"maniguard":[4,7],"roboengineer":[2,15]},"claude_code-opus_5_5-high":{"behavior-1k":[5,71],"robolab":[1,6],"robocasa":[2,2],"robocasa365":[1,5],"robocasa-gr1":[1,1],"metaworldplus":[2,2],"vlabench":[1,2],"humanoidbench":[0,8],"molmospaces":[2,2],"kinder":[0,4],"robopaint":[2,22],"robotwin-2":[3,3],"mujoco-playground":[0,7],"dextoolbench":[0,20],"dexjoco":[2,2],"myosuite":[3,13],"roboarc":[15,58],"robodojo":[18,37],"pandadyn":[1,10],"bigym2":[2,10],"yamlab":[2,2],"libero-max":[2,2],"embodied-olympics-fun":[16,56],"embodied-olympics-match":[3,13],"ambench":[1,2],"embodiedswe":[6,12],"embodied-safety-exam":[19,50],"dexhand":[2,23],"ispatialgym":[3,8],"robowits":[4,6],"maniguard":[7,7],"roboengineer":[4,15]},"codex-gpt6_1_sol-high":{"behavior-1k":[13,71],"robolab":[2,6],"robocasa":[2,2],"robocasa365":[1,5],"robocasa-gr1":[0,1],"metaworldplus":[2,2],"vlabench":[1,2],"humanoidbench":[0,8],"molmospaces":[2,2],"kinder":[1,4],"robopaint":[6,22],"robotwin-2":[2,3],"mujoco-playground":[3,7],"dextoolbench":[1,20],"dexjoco":[0,2],"myosuite":[3,13],"roboarc":[14,58],"robodojo":[16,37],"pandadyn":[4,10],"bigym2":[0,10],"yamlab":[1,2],"libero-max":[2,2],"embodied-olympics-fun":[20,56],"embodied-olympics-match":[6,13],"ambench":[2,2],"embodiedswe":[5,12],"embodied-safety-exam":[29,50],"dexhand":[9,23],"ispatialgym":[5,8],"robowits":[3,6],"maniguard":[5,7],"roboengineer":[3,15]},"codex-gpt6_2_astra-high":{"behavior-1k":[40,71],"robolab":[2,6],"robocasa":[2,2],"robocasa365":[3,5],"robocasa-gr1":[1,1],"metaworldplus":[2,2],"vlabench":[1,2],"humanoidbench":[1,8],"molmospaces":[2,2],"kinder":[2,4],"robopaint":[21,22],"robotwin-2":[3,3],"mujoco-playground":[7,7],"dextoolbench":[4,20],"dexjoco":[2,2],"myosuite":[7,13],"roboarc":[32,58],"robodojo":[22,37],"pandadyn":[5,10],"bigym2":[7,10],"yamlab":[2,2],"libero-max":[2,2],"embodied-olympics-fun":[27,56],"embodied-olympics-match":[9,13],"ambench":[1,2],"embodiedswe":[8,12],"embodied-safety-exam":[27,50],"dexhand":[20,23],"ispatialgym":[7,8],"robowits":[3,6],"maniguard":[4,7],"roboengineer":[3,15]},"codex-gpt6_astra-high":{"behavior-1k":[14,71],"robolab":[0,6],"robocasa":[0,2],"robocasa365":[1,5],"robocasa-gr1":[0,1],"metaworldplus":[1,2],"vlabench":[0,2],"humanoidbench":[1,8],"molmospaces":[2,2],"kinder":[0,4],"robopaint":[7,22],"robotwin-2":[1,3],"mujoco-playground":[4,7],"dextoolbench":[2,20],"dexjoco":[1,2],"myosuite":[3,13],"roboarc":[15,58],"robodojo":[21,37],"pandadyn":[3,10],"bigym2":[4,10],"yamlab":[1,2],"libero-max":[2,2],"embodied-olympics-fun":[20,56],"embodied-olympics-match":[6,13],"ambench":[2,2],"embodiedswe":[5,12],"embodied-safety-exam":[32,50],"dexhand":[13,23],"ispatialgym":[4,8],"robowits":[2,6],"maniguard":[3,7],"roboengineer":[5,15]},"dsh-deepseek_v4_1_flash-high":{"behavior-1k":[0,71],"robolab":[0,6],"robocasa":[1,2],"robocasa365":[0,5],"robocasa-gr1":[0,1],"metaworldplus":[1,2],"vlabench":[0,2],"humanoidbench":[0,8],"molmospaces":[1,2],"kinder":[0,4],"robopaint":[0,22],"robotwin-2":[0,3],"mujoco-playground":[0,7],"dextoolbench":[0,20],"dexjoco":[0,2],"myosuite":[0,13],"roboarc":[2,58],"robodojo":[3,37],"pandadyn":[0,10],"bigym2":[0,10],"yamlab":[0,2],"libero-max":[0,2],"embodied-olympics-fun":[2,56],"embodied-olympics-match":[0,13],"ambench":[2,2],"embodiedswe":[0,12],"embodied-safety-exam":[2,50],"dexhand":[0,23],"ispatialgym":[1,8],"robowits":[0,6],"maniguard":[0,7],"roboengineer":[2,15]}}},"privileged":{"tests":416,"scores":{"antigravity_cli-gemini_3_8_flash-high":[102,416,416],"antigravity_cli-gemini_4_argon-high":[188,416,416],"claude_code-fable_5_2-high":[337,416,416],"claude_code-glm_5_3_flash-high":[44,416,416],"claude_code-kimi_3_1_pro-high":[165,416,416],"claude_code-opus_5_5-high":[257,416,416],"codex-gpt6_1_sol-high":[309,416,416],"codex-gpt6_2_astra-high":[395,416,416],"codex-gpt6_astra-high":[333,416,416],"dsh-deepseek_v4_1_flash-high":[73,416,416]},"suites":{"antigravity_cli-gemini_3_8_flash-high":{"behavior-1k":[2,71],"robolab":[2,6],"robocasa":[2,2],"robocasa365":[0,5],"robocasa-gr1":[0,1],"metaworldplus":[2,2],"vlabench":[0,2],"humanoidbench":[0,8],"molmospaces":[2,2],"kinder":[0,4],"robopaint":[4,22],"robotwin-2":[1,3],"mujoco-playground":[3,7],"dextoolbench":[2,20],"dexjoco":[2,2],"myosuite":[2,13],"roboarc":[5,58],"robodojo":[9,37],"pandadyn":[2,10],"bigym2":[6,10],"yamlab":[1,2],"libero-max":[2,2],"embodied-olympics-fun":[22,56],"embodied-olympics-match":[9,13],"ambench":[2,2],"embodiedswe":[5,12],"dexhand":[5,23],"ispatialgym":[3,8],"robowits":[4,6],"maniguard":[3,7]},"antigravity_cli-gemini_4_argon-high":{"behavior-1k":[6,71],"robolab":[1,6],"robocasa":[2,2],"robocasa365":[3,5],"robocasa-gr1":[1,1],"metaworldplus":[2,2],"vlabench":[1,2],"humanoidbench":[2,8],"molmospaces":[2,2],"kinder":[0,4],"robopaint":[15,22],"robotwin-2":[3,3],"mujoco-playground":[4,7],"dextoolbench":[1,20],"dexjoco":[1,2],"myosuite":[0,13],"roboarc":[27,58],"robodojo":[24,37],"pandadyn":[5,10],"bigym2":[7,10],"yamlab":[2,2],"libero-max":[2,2],"embodied-olympics-fun":[32,56],"embodied-olympics-match":[12,13],"ambench":[0,2],"embodiedswe":[9,12],"dexhand":[8,23],"ispatialgym":[4,8],"robowits":[5,6],"maniguard":[7,7]},"claude_code-fable_5_2-high":{"behavior-1k":[36,71],"robolab":[5,6],"robocasa":[2,2],"robocasa365":[5,5],"robocasa-gr1":[1,1],"metaworldplus":[2,2],"vlabench":[2,2],"humanoidbench":[3,8],"molmospaces":[2,2],"kinder":[2,4],"robopaint":[20,22],"robotwin-2":[3,3],"mujoco-playground":[7,7],"dextoolbench":[10,20],"dexjoco":[2,2],"myosuite":[6,13],"roboarc":[53,58],"robodojo":[36,37],"pandadyn":[10,10],"bigym2":[10,10],"yamlab":[2,2],"libero-max":[2,2],"embodied-olympics-fun":[49,56],"embodied-olympics-match":[12,13],"ambench":[2,2],"embodiedswe":[12,12],"dexhand":[22,23],"ispatialgym":[7,8],"robowits":[5,6],"maniguard":[7,7]},"claude_code-glm_5_3_flash-high":{"behavior-1k":[1,71],"robolab":[1,6],"robocasa":[2,2],"robocasa365":[0,5],"robocasa-gr1":[0,1],"metaworldplus":[2,2],"vlabench":[0,2],"humanoidbench":[0,8],"molmospaces":[2,2],"kinder":[1,4],"robopaint":[0,22],"robotwin-2":[0,3],"mujoco-playground":[1,7],"dextoolbench":[0,20],"dexjoco":[0,2],"myosuite":[0,13],"roboarc":[1,58],"robodojo":[6,37],"pandadyn":[1,10],"bigym2":[0,10],"yamlab":[0,2],"libero-max":[0,2],"embodied-olympics-fun":[16,56],"embodied-olympics-match":[5,13],"ambench":[2,2],"embodiedswe":[2,12],"dexhand":[0,23],"ispatialgym":[0,8],"robowits":[1,6],"maniguard":[0,7]},"claude_code-kimi_3_1_pro-high":{"behavior-1k":[13,71],"robolab":[3,6],"robocasa":[2,2],"robocasa365":[0,5],"robocasa-gr1":[0,1],"metaworldplus":[2,2],"vlabench":[1,2],"humanoidbench":[1,8],"molmospaces":[2,2],"kinder":[0,4],"robopaint":[14,22],"robotwin-2":[1,3],"mujoco-playground":[5,7],"dextoolbench":[0,20],"dexjoco":[1,2],"myosuite":[3,13],"roboarc":[13,58],"robodojo":[19,37],"pandadyn":[7,10],"bigym2":[8,10],"yamlab":[2,2],"libero-max":[2,2],"embodied-olympics-fun":[29,56],"embodied-olympics-match":[8,13],"ambench":[2,2],"embodiedswe":[6,12],"dexhand":[10,23],"ispatialgym":[2,8],"robowits":[4,6],"maniguard":[5,7]},"claude_code-opus_5_5-high":{"behavior-1k":[22,71],"robolab":[3,6],"robocasa":[2,2],"robocasa365":[4,5],"robocasa-gr1":[1,1],"metaworldplus":[2,2],"vlabench":[1,2],"humanoidbench":[2,8],"molmospaces":[2,2],"kinder":[1,4],"robopaint":[11,22],"robotwin-2":[3,3],"mujoco-playground":[5,7],"dextoolbench":[0,20],"dexjoco":[2,2],"myosuite":[5,13],"roboarc":[43,58],"robodojo":[27,37],"pandadyn":[10,10],"bigym2":[5,10],"yamlab":[2,2],"libero-max":[2,2],"embodied-olympics-fun":[45,56],"embodied-olympics-match":[12,13],"ambench":[2,2],"embodiedswe":[10,12],"dexhand":[16,23],"ispatialgym":[5,8],"robowits":[5,6],"maniguard":[7,7]},"codex-gpt6_1_sol-high":{"behavior-1k":[38,71],"robolab":[4,6],"robocasa":[2,2],"robocasa365":[5,5],"robocasa-gr1":[1,1],"metaworldplus":[2,2],"vlabench":[2,2],"humanoidbench":[3,8],"molmospaces":[2,2],"kinder":[2,4],"robopaint":[20,22],"robotwin-2":[3,3],"mujoco-playground":[7,7],"dextoolbench":[10,20],"dexjoco":[2,2],"myosuite":[7,13],"roboarc":[40,58],"robodojo":[29,37],"pandadyn":[10,10],"bigym2":[9,10],"yamlab":[2,2],"libero-max":[2,2],"embodied-olympics-fun":[45,56],"embodied-olympics-match":[12,13],"ambench":[2,2],"embodiedswe":[11,12],"dexhand":[18,23],"ispatialgym":[7,8],"robowits":[5,6],"maniguard":[7,7]},"codex-gpt6_2_astra-high":{"behavior-1k":[67,71],"robolab":[6,6],"robocasa":[2,2],"robocasa365":[5,5],"robocasa-gr1":[1,1],"metaworldplus":[2,2],"vlabench":[2,2],"humanoidbench":[5,8],"molmospaces":[2,2],"kinder":[3,4],"robopaint":[22,22],"robotwin-2":[3,3],"mujoco-playground":[7,7],"dextoolbench":[17,20],"dexjoco":[2,2],"myosuite":[8,13],"roboarc":[58,58],"robodojo":[37,37],"pandadyn":[10,10],"bigym2":[10,10],"yamlab":[2,2],"libero-max":[2,2],"embodied-olympics-fun":[52,56],"embodied-olympics-match":[12,13],"ambench":[2,2],"embodiedswe":[12,12],"dexhand":[23,23],"ispatialgym":[8,8],"robowits":[6,6],"maniguard":[7,7]},"codex-gpt6_astra-high":{"behavior-1k":[36,71],"robolab":[5,6],"robocasa":[2,2],"robocasa365":[5,5],"robocasa-gr1":[1,1],"metaworldplus":[2,2],"vlabench":[2,2],"humanoidbench":[3,8],"molmospaces":[2,2],"kinder":[2,4],"robopaint":[22,22],"robotwin-2":[3,3],"mujoco-playground":[7,7],"dextoolbench":[13,20],"dexjoco":[2,2],"myosuite":[6,13],"roboarc":[47,58],"robodojo":[35,37],"pandadyn":[10,10],"bigym2":[10,10],"yamlab":[2,2],"libero-max":[2,2],"embodied-olympics-fun":[49,56],"embodied-olympics-match":[11,13],"ambench":[2,2],"embodiedswe":[11,12],"dexhand":[22,23],"ispatialgym":[7,8],"robowits":[5,6],"maniguard":[7,7]},"dsh-deepseek_v4_1_flash-high":{"behavior-1k":[3,71],"robolab":[0,6],"robocasa":[2,2],"robocasa365":[0,5],"robocasa-gr1":[0,1],"metaworldplus":[2,2],"vlabench":[1,2],"humanoidbench":[0,8],"molmospaces":[2,2],"kinder":[0,4],"robopaint":[1,22],"robotwin-2":[2,3],"mujoco-playground":[3,7],"dextoolbench":[1,20],"dexjoco":[1,2],"myosuite":[3,13],"roboarc":[5,58],"robodojo":[7,37],"pandadyn":[2,10],"bigym2":[0,10],"yamlab":[0,2],"libero-max":[2,2],"embodied-olympics-fun":[18,56],"embodied-olympics-match":[8,13],"ambench":[2,2],"embodiedswe":[2,12],"dexhand":[0,23],"ispatialgym":[3,8],"robowits":[1,6],"maniguard":[2,7]}}},"realtime":{"tests":54,"scores":{"antigravity_cli-gemini_3_8_flash-high":[1,54,54],"antigravity_cli-gemini_4_argon-high":[5,54,54],"claude_code-fable_5_2-high":[22,54,54],"claude_code-glm_5_3_flash-high":[0,54,54],"claude_code-kimi_3_1_pro-high":[4,54,54],"claude_code-opus_5_5-high":[4,54,54],"codex-gpt6_1_sol-high":[13,54,54],"codex-gpt6_2_astra-high":[22,54,54],"codex-gpt6_astra-high":[17,54,54],"dsh-deepseek_v4_1_flash-high":[1,54,54]},"suites":{"antigravity_cli-gemini_3_8_flash-high":{"humanoidbench":[0,8],"mujoco-playground":[0,1],"pandadyn":[0,8],"bigym2":[0,10],"embodied-olympics-fun":[0,11],"embodied-olympics-match":[1,12],"embodied-safety-exam":[0,4]},"antigravity_cli-gemini_4_argon-high":{"humanoidbench":[0,8],"mujoco-playground":[0,1],"pandadyn":[0,8],"bigym2":[0,10],"embodied-olympics-fun":[0,11],"embodied-olympics-match":[2,12],"embodied-safety-exam":[3,4]},"claude_code-fable_5_2-high":{"humanoidbench":[1,8],"mujoco-playground":[0,1],"pandadyn":[6,8],"bigym2":[1,10],"embodied-olympics-fun":[6,11],"embodied-olympics-match":[5,12],"embodied-safety-exam":[3,4]},"claude_code-glm_5_3_flash-high":{"humanoidbench":[0,8],"mujoco-playground":[0,1],"pandadyn":[0,8],"bigym2":[0,10],"embodied-olympics-fun":[0,11],"embodied-olympics-match":[0,12],"embodied-safety-exam":[0,4]},"claude_code-kimi_3_1_pro-high":{"humanoidbench":[0,8],"mujoco-playground":[0,1],"pandadyn":[0,8],"bigym2":[0,10],"embodied-olympics-fun":[2,11],"embodied-olympics-match":[2,12],"embodied-safety-exam":[0,4]},"claude_code-opus_5_5-high":{"humanoidbench":[0,8],"mujoco-playground":[0,1],"pandadyn":[1,8],"bigym2":[1,10],"embodied-olympics-fun":[1,11],"embodied-olympics-match":[0,12],"embodied-safety-exam":[1,4]},"codex-gpt6_1_sol-high":{"humanoidbench":[0,8],"mujoco-playground":[1,1],"pandadyn":[1,8],"bigym2":[0,10],"embodied-olympics-fun":[6,11],"embodied-olympics-match":[4,12],"embodied-safety-exam":[1,4]},"codex-gpt6_2_astra-high":{"humanoidbench":[1,8],"mujoco-playground":[1,1],"pandadyn":[6,8],"bigym2":[1,10],"embodied-olympics-fun":[4,11],"embodied-olympics-match":[7,12],"embodied-safety-exam":[2,4]},"codex-gpt6_astra-high":{"humanoidbench":[1,8],"mujoco-playground":[1,1],"pandadyn":[2,8],"bigym2":[1,10],"embodied-olympics-fun":[6,11],"embodied-olympics-match":[4,12],"embodied-safety-exam":[2,4]},"dsh-deepseek_v4_1_flash-high":{"humanoidbench":[0,8],"mujoco-playground":[0,1],"pandadyn":[0,8],"bigym2":[0,10],"embodied-olympics-fun":[1,11],"embodied-olympics-match":[0,12],"embodied-safety-exam":[0,4]}}}},"shown_sets":{"antigravity_cli-gemini_3_8_flash-high":{"behavior-1k":0,"robolab":0,"robocasa":0,"robocasa365":0,"robocasa-gr1":0,"metaworldplus":0,"vlabench":0,"humanoidbench":0,"molmospaces":0,"kinder":0,"robopaint":0,"robotwin-2":0,"mujoco-playground":0,"dextoolbench":0,"dexjoco":0,"myosuite":0,"roboarc":0,"robodojo":0,"pandadyn":0,"bigym2":0,"yamlab":0,"libero-max":0,"embodied-olympics-fun":0,"embodied-olympics-match":0,"ambench":0,"embodiedswe":0,"embodied-safety-exam":0,"dexhand":0,"ispatialgym":0,"robowits":0,"maniguard":0,"roboengineer":0},"antigravity_cli-gemini_4_argon-high":{"behavior-1k":0,"robolab":0,"robocasa":0,"robocasa365":0,"robocasa-gr1":0,"metaworldplus":0,"vlabench":0,"humanoidbench":0,"molmospaces":0,"kinder":0,"robopaint":0,"robotwin-2":0,"mujoco-playground":0,"dextoolbench":0,"dexjoco":0,"myosuite":0,"roboarc":0,"robodojo":0,"pandadyn":0,"bigym2":0,"yamlab":0,"libero-max":0,"embodied-olympics-fun":0,"embodied-olympics-match":0,"ambench":0,"embodiedswe":0,"embodied-safety-exam":0,"dexhand":0,"ispatialgym":0,"robowits":0,"maniguard":0,"roboengineer":0},"claude_code-fable_5_2-high":{"behavior-1k":0,"robolab":0,"robocasa":0,"robocasa365":0,"robocasa-gr1":0,"metaworldplus":0,"vlabench":0,"humanoidbench":0,"molmospaces":0,"kinder":0,"robopaint":0,"robotwin-2":0,"mujoco-playground":0,"dextoolbench":0,"dexjoco":0,"myosuite":0,"roboarc":0,"robodojo":0,"pandadyn":0,"bigym2":0,"yamlab":0,"libero-max":0,"embodied-olympics-fun":0,"embodied-olympics-match":0,"ambench":0,"embodiedswe":0,"embodied-safety-exam":0,"dexhand":0,"ispatialgym":0,"robowits":0,"maniguard":0,"roboengineer":0},"claude_code-glm_5_3_flash-high":{"behavior-1k":0,"robolab":0,"robocasa":0,"robocasa365":0,"robocasa-gr1":0,"metaworldplus":0,"vlabench":0,"humanoidbench":0,"molmospaces":0,"kinder":0,"robopaint":0,"robotwin-2":0,"mujoco-playground":0,"dextoolbench":0,"dexjoco":0,"myosuite":0,"roboarc":0,"robodojo":0,"pandadyn":0,"bigym2":0,"yamlab":0,"libero-max":0,"embodied-olympics-fun":0,"embodied-olympics-match":0,"ambench":0,"embodiedswe":0,"embodied-safety-exam":0,"dexhand":0,"ispatialgym":0,"robowits":0,"maniguard":0,"roboengineer":0},"claude_code-kimi_3_1_pro-high":{"behavior-1k":0,"robolab":0,"robocasa":0,"robocasa365":0,"robocasa-gr1":0,"metaworldplus":0,"vlabench":0,"humanoidbench":0,"molmospaces":0,"kinder":0,"robopaint":0,"robotwin-2":0,"mujoco-playground":0,"dextoolbench":0,"dexjoco":0,"myosuite":0,"roboarc":0,"robodojo":0,"pandadyn":0,"bigym2":0,"yamlab":0,"libero-max":0,"embodied-olympics-fun":0,"embodied-olympics-match":0,"ambench":0,"embodiedswe":0,"embodied-safety-exam":0,"dexhand":0,"ispatialgym":0,"robowits":0,"maniguard":0,"roboengineer":0},"claude_code-opus_5_5-high":{"behavior-1k":0,"robolab":0,"robocasa":0,"robocasa365":0,"robocasa-gr1":0,"metaworldplus":0,"vlabench":0,"humanoidbench":0,"molmospaces":0,"kinder":0,"robopaint":0,"robotwin-2":0,"mujoco-playground":0,"dextoolbench":0,"dexjoco":0,"myosuite":0,"roboarc":0,"robodojo":0,"pandadyn":0,"bigym2":0,"yamlab":0,"libero-max":0,"embodied-olympics-fun":0,"embodied-olympics-match":0,"ambench":0,"embodiedswe":0,"embodied-safety-exam":0,"dexhand":0,"ispatialgym":0,"robowits":0,"maniguard":0,"roboengineer":0},"codex-gpt6_1_sol-high":{"behavior-1k":0,"robolab":0,"robocasa":0,"robocasa365":0,"robocasa-gr1":0,"metaworldplus":0,"vlabench":0,"humanoidbench":0,"molmospaces":0,"kinder":0,"robopaint":0,"robotwin-2":0,"mujoco-playground":0,"dextoolbench":0,"dexjoco":0,"myosuite":0,"roboarc":0,"robodojo":0,"pandadyn":0,"bigym2":0,"yamlab":0,"libero-max":0,"embodied-olympics-fun":0,"embodied-olympics-match":0,"ambench":0,"embodiedswe":0,"embodied-safety-exam":0,"dexhand":0,"ispatialgym":0,"robowits":0,"maniguard":0,"roboengineer":0},"codex-gpt6_2_astra-high":{"behavior-1k":0,"robolab":0,"robocasa":0,"robocasa365":0,"robocasa-gr1":0,"metaworldplus":0,"vlabench":0,"humanoidbench":0,"molmospaces":0,"kinder":0,"robopaint":0,"robotwin-2":0,"mujoco-playground":0,"dextoolbench":0,"dexjoco":0,"myosuite":0,"roboarc":0,"robodojo":0,"pandadyn":0,"bigym2":0,"yamlab":0,"libero-max":0,"embodied-olympics-fun":0,"embodied-olympics-match":0,"ambench":0,"embodiedswe":0,"embodied-safety-exam":0,"dexhand":0,"ispatialgym":0,"robowits":0,"maniguard":0,"roboengineer":0},"codex-gpt6_astra-high":{"behavior-1k":0,"robolab":0,"robocasa":0,"robocasa365":0,"robocasa-gr1":0,"metaworldplus":0,"vlabench":0,"humanoidbench":0,"molmospaces":0,"kinder":0,"robopaint":0,"robotwin-2":0,"mujoco-playground":0,"dextoolbench":0,"dexjoco":0,"myosuite":0,"roboarc":0,"robodojo":0,"pandadyn":0,"bigym2":0,"yamlab":0,"libero-max":0,"embodied-olympics-fun":0,"embodied-olympics-match":0,"ambench":0,"embodiedswe":0,"embodied-safety-exam":0,"dexhand":0,"ispatialgym":0,"robowits":0,"maniguard":0,"roboengineer":0},"dsh-deepseek_v4_1_flash-high":{"behavior-1k":0,"robolab":0,"robocasa":0,"robocasa365":0,"robocasa-gr1":0,"metaworldplus":0,"vlabench":0,"humanoidbench":0,"molmospaces":0,"kinder":0,"robopaint":0,"robotwin-2":0,"mujoco-playground":0,"dextoolbench":0,"dexjoco":0,"myosuite":0,"roboarc":0,"robodojo":0,"pandadyn":0,"bigym2":0,"yamlab":0,"libero-max":0,"embodied-olympics-fun":0,"embodied-olympics-match":0,"ambench":0,"embodiedswe":0,"embodied-safety-exam":0,"dexhand":0,"ispatialgym":0,"robowits":0,"maniguard":0,"roboengineer":0}},"core_off":["codex-gpt6_luna-xhigh"]}},"contribute":{"recognition":{"draft":true,"lead":"Every task names its author, and every suite credits its upstream benchmark. A proposal for the maintainers:","credits":[{"what":"A suite accepted into the exam: proposal, tasks, image and evidence","credits":15},{"what":"A family of tasks added to a suite already in the exam","credits":8},{"what":"A human review of one family (PROTOCOL §2.7)","credits":3},{"what":"A verified exploit or task bug, with its fix merged","credits":3},{"what":"A full evaluation run of the exam, logs delivered","credits":5}],"threshold":15,"threshold_note":"Contributors at or above the threshold are invited as authors; core maintainers are listed first."}},"hero":{"lead":"A Terminal-Bench for coding agents in robotics.","tiles":[{"suite":"embodied-olympics-fun","id":"free_throw","title":"Free Throw","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/free_throw.jpg?v=59a8c7c5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/wall/free_throw.mp4?v=8d6ede84"},{"suite":"roboarc","id":"interlocked_claws_05","title":"Interlocked Claws 05","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/interlocked_claws_05.jpg?v=27ce3fd0","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/wall/interlocked_claws_05.mp4?v=8dc77e12"},{"suite":"behavior-1k","id":"setting_the_table","title":"Setting The Table","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/setting_the_table.jpg?v=71c83842","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/wall/setting_the_table.mp4?v=5bf84398"},{"suite":"myosuite","id":"osl_run","title":"Osl Run","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/osl_run.jpg?v=e0bcd8ed","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/wall/osl_run.mp4?v=7cf9fd2a"},{"suite":"dexjoco","id":"bimanual_hanoi","title":"Bimanual Hanoi","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/bimanual_hanoi.jpg?v=3b413ab9","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/wall/bimanual_hanoi.mp4?v=8acfccb7"},{"suite":"dexhand","id":"book_page_turn","title":"Book Page Turn","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/book_page_turn.jpg?v=8c4e5405","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/book_page_turn.mp4?v=1c5215b8"},{"suite":"mujoco-playground","id":"aloha_single_peg_insertion","title":"Aloha Single Peg Insertion","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/aloha_single_peg_insertion.jpg?v=74a75082","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/wall/aloha_single_peg_insertion.mp4?v=e42b1142"},{"suite":"humanoidbench","id":"loco_hurdle","title":"Locomotion · Hurdle","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/loco_hurdle.jpg?v=066370cd","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/wall/loco_hurdle.mp4?v=d14bea5c"},{"suite":"kinder","id":"constrained_cupboard","title":"Dynamic3D · Constrained cupboard","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/constrained_cupboard.jpg?v=a91f3451","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/wall/constrained_cupboard.mp4?v=88d17ac8"},{"suite":"pandadyn","id":"d09_lever_launch_catch_reload","title":"D09 Lever Launch Catch Reload","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/d09_lever_launch_catch_reload.jpg?v=4dbaea9f","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/wall/d09_lever_launch_catch_reload.mp4?v=e5899618"},{"suite":"robocasa-gr1","id":"can_to_drawer_close","title":"GR1 · PnPCanToDrawerClose","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/can_to_drawer_close.jpg?v=f21c1747","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/wall/can_to_drawer_close.mp4?v=9862f05d"},{"suite":"ambench","id":"lemon_harvesting","title":"Lemon Harvesting","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/lemon_harvesting.jpg?v=11dedb1c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/wall/lemon_harvesting.mp4?v=9aa8f9aa"},{"suite":"robowits","id":"dominos","title":"Dominos","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/dominos.jpg?v=e509c63d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/dominos.mp4?v=fa6e3f37"},{"suite":"robopaint","id":"oil_starrynight_i01","title":"Oil · Starrynight","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/oil_starrynight_i01.jpg?v=3a6f8cf6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/wall/oil_starrynight_i01.mp4?v=523b2d10"},{"suite":"robotwin-2","id":"hanging_mug","title":"Hanging mug","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/hanging_mug.jpg?v=8f5f65f8","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/wall/hanging_mug.mp4?v=aacba03c"},{"suite":"dextoolbench","id":"hammer_claw_hammer_swing_side","title":"Claw hammer · swing side","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/hammer_claw_hammer_swing_side.jpg?v=972b161a","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dextoolbench/wall/hammer_claw_hammer_swing_side.mp4?v=35d61fdf"},{"suite":"robocasa365","id":"open_dishwasher","title":"Atomic · OpenDishwasher","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/open_dishwasher.jpg?v=f3768d4e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/wall/open_dishwasher.mp4?v=b0946072"},{"suite":"maniguard","id":"cabinet_pickup_i22","title":"Cabinet Pickup I22","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/cabinet_pickup_i22.jpg?v=2e63bea5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/wall/cabinet_pickup_i22.mp4?v=e0f8a8b6"},{"suite":"metaworldplus","id":"basketball","title":"Meta-World · Basketball","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/basketball.jpg?v=85b4dcb2","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/metaworldplus/wall/basketball.mp4?v=4e0add8c"},{"suite":"vlabench","id":"hammer_nail_and_hang_picture","title":"Manipulation · hammer_nail_and_hang_picture","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/hammer_nail_and_hang_picture.jpg?v=ce7c5b3c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/wall/hammer_nail_and_hang_picture.mp4?v=d44d5979"},{"suite":"yamlab","id":"hang_mug_on_tree_i10","title":"Hang Mug On Tree I10","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/hang_mug_on_tree_i10.jpg?v=253aa2a6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/hang_mug_on_tree_i10.mp4?v=ece19f89"},{"suite":"robolab","id":"reorient_red_mug","title":"Reorient Red Mug","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/reorient_red_mug.jpg?v=4b65d22f","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/wall/reorient_red_mug.mp4?v=4fc1d024"},{"suite":"robodojo","id":"match_and_pick_from_conveyor","title":"Match And Pick From Conveyor","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/match_and_pick_from_conveyor.jpg?v=85c4a487","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/wall/match_and_pick_from_conveyor.mp4?v=66c98d64"},{"suite":"embodied-safety-exam","id":"c04_i01","title":"Set the table on a place mat · C04 I01","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/c04_i01.jpg?v=63d8a7db","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robosafe/wall/c04_i01.mp4?v=df0f7cfa"},{"suite":"autobio","id":"screw_tighten","title":"Screw Tighten","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/screw_tighten.jpg?v=14ec9829","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/autobio/screw_tighten.mp4?v=702f05fd"},{"suite":"embodied-olympics-fun","id":"catch_and_shoot","title":"Catch And Shoot","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/catch_and_shoot.jpg?v=d6a1cbd9","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/wall/catch_and_shoot.mp4?v=ceb6f4e9"},{"suite":"roboarc","id":"knot_untying_03","title":"Knot Untying 03","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/knot_untying_03.jpg?v=16dbd6c7","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/knot_untying_03.mp4?v=647c5248"},{"suite":"behavior-1k","id":"cook_brussels_sprouts","title":"Cook Brussels Sprouts","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/cook_brussels_sprouts.jpg?v=e8c647d8","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/wall/cook_brussels_sprouts.mp4?v=d29a6c72"},{"suite":"myosuite","id":"baoding","title":"Baoding","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/baoding.jpg?v=10ecbc1e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/wall/baoding.mp4?v=147d4e07"},{"suite":"dexjoco","id":"water_plant","title":"Water Plant","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/water_plant.jpg?v=2a53a472","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/wall/water_plant.mp4?v=627e83aa"},{"suite":"dexhand","id":"jar_lid_open","title":"Jar Lid Open","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/jar_lid_open.jpg?v=ef6561b7","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/jar_lid_open.mp4?v=f9f55e2c"},{"suite":"mujoco-playground","id":"go1_handstand","title":"Go1 Handstand","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/go1_handstand.jpg?v=089451b6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/wall/go1_handstand.mp4?v=67db9264"},{"suite":"humanoidbench","id":"loco_balance_hard","title":"Locomotion · Balance hard","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/loco_balance_hard.jpg?v=71b6721d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/wall/loco_balance_hard.mp4?v=e07cc468"},{"suite":"kinder","id":"sweep_into_drawer","title":"Dynamic3D · Sweep into drawer","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/sweep_into_drawer.jpg?v=b65dfc46","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/wall/sweep_into_drawer.mp4?v=00e54014"},{"suite":"pandadyn","id":"s09_toss_recatch_delivery","title":"S09 Toss Recatch Delivery","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/s09_toss_recatch_delivery.jpg?v=7633d270","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/pandadyn/s09_toss_recatch_delivery.mp4?v=17ba0dba"},{"suite":"robocasa-gr1","id":"milk_to_microwave_close","title":"GR1 · PnPMilkToMicrowaveClose","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/milk_to_microwave_close.jpg?v=6d308c9c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/milk_to_microwave_close.mp4?v=751f3600"},{"suite":"ambench","id":"open_door","title":"Open Door","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/open_door.jpg?v=56c1f167","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/open_door.mp4?v=7cd079d9"},{"suite":"robowits","id":"ball_into_bottle","title":"Ball Into Bottle","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/ball_into_bottle.jpg?v=77854660","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/wall/ball_into_bottle.mp4?v=fa157d48"},{"suite":"robopaint","id":"acrylic_greatwave","title":"Acrylic · Greatwave","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/acrylic_greatwave.jpg?v=a76da25a","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/acrylic_greatwave.mp4?v=b9bc2b76"},{"suite":"robocasa365","id":"close_stand_mixer_head","title":"Atomic · CloseStandMixerHead","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/close_stand_mixer_head.jpg?v=80e38c99","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/wall/close_stand_mixer_head.mp4?v=330d2638"},{"suite":"maniguard","id":"jar_transport_i02","title":"Jar Transport I02","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/jar_transport_i02.jpg?v=88cba88b","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/wall/jar_transport_i02.mp4?v=8a002491"},{"suite":"vlabench","id":"texas_holdem","title":"Manipulation · texas_holdem","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/texas_holdem.jpg?v=72bb5c84","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/wall/texas_holdem.mp4?v=3d71f834"},{"suite":"yamlab","id":"put_pot_on_cooktop_i03","title":"Put Pot On Cooktop I03","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/put_pot_on_cooktop_i03.jpg?v=89736e02","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/yamlab/put_pot_on_cooktop_i03.mp4?v=a86fd886"},{"suite":"robolab","id":"block_stacking_specified_order","title":"Block Stacking Specified Order","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/block_stacking_specified_order.jpg?v=c1ca5ce6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/wall/block_stacking_specified_order.mp4?v=c578cc6e"},{"suite":"robodojo","id":"deposit_coin","title":"Deposit Coin","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/deposit_coin.jpg?v=1b746803","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robodojo-agent/wall/deposit_coin.mp4?v=075baf31"},{"suite":"embodied-olympics-fun","id":"backflip","title":"Backflip","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/backflip.jpg?v=131b4cea","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/wall/backflip.mp4?v=a6577e73"},{"suite":"roboarc","id":"buzz_wire_3d_06","title":"Buzz Wire 3D 06","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/buzz_wire_3d_06.jpg?v=fe029c77","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/buzz_wire_3d_06.mp4?v=343cc0d4"},{"suite":"behavior-1k","id":"putting_up_Christmas_decorations_inside","title":"Putting Up Christmas Decorations Inside","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/putting_up_Christmas_decorations_inside.jpg?v=d713afec","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/putting_up_Christmas_decorations_inside.mp4?v=8a6999bc"},{"suite":"myosuite","id":"soccer","title":"Soccer","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/soccer.jpg?v=33c394c5","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/wall/soccer.mp4?v=30e2889c"},{"suite":"dexjoco","id":"bimanual_photograph","title":"Bimanual Photograph","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/bimanual_photograph.jpg?v=7e2389c4","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/wall/bimanual_photograph.mp4?v=d0f18bc7"},{"suite":"dexhand","id":"essential_pen_regrasp","title":"Essential Pen Regrasp","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/essential_pen_regrasp.jpg?v=83a21b11","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/essential_pen_regrasp.mp4?v=dd57abea"},{"suite":"ambench","id":"ndt","title":"Ndt","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/ndt.jpg?v=31701a63","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/ambench/ndt.mp4?v=7f85088c"},{"suite":"robowits","id":"stand_bulb","title":"Stand Bulb","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/stand_bulb.jpg?v=be6d84c6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/stand_bulb.mp4?v=b406c410"},{"suite":"robopaint","id":"caoshu_zhongtiangechuifen","title":"Caoshu · Zhongtiangechuifen","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/caoshu_zhongtiangechuifen.jpg?v=d3990ada","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/caoshu_zhongtiangechuifen.mp4?v=9327657b"},{"suite":"robocasa365","id":"oven_broil_fish","title":"Composite · OvenBroilFish","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/oven_broil_fish.jpg?v=ed2fba83","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/wall/oven_broil_fish.mp4?v=a416afb0"},{"suite":"maniguard","id":"lid_transport_i07","title":"Lid Transport I07","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/lid_transport_i07.jpg?v=b8ca07a7","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/maniguard/wall/lid_transport_i07.mp4?v=34f94aa9"},{"suite":"vlabench","id":"insert_bloom_flower","title":"Manipulation · insert_bloom_flower","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/insert_bloom_flower.jpg?v=d2cead6e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/insert_bloom_flower.mp4?v=0c3123ad"},{"suite":"robolab","id":"take_mugs_off_of_shelf","title":"Take Mugs Off Of Shelf","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/take_mugs_off_of_shelf.jpg?v=dda96679","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/wall/take_mugs_off_of_shelf.mp4?v=a3fb8d31"},{"suite":"embodied-olympics-fun","id":"high_jump","title":"High Jump","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/high_jump.jpg?v=bfa326f6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/wall/high_jump.mp4?v=ea8a771e"},{"suite":"roboarc","id":"blanket_fold_03","title":"Blanket Fold 03","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/blanket_fold_03.jpg?v=38c9752f","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/blanket_fold_03.mp4?v=e13f258b"},{"suite":"behavior-1k","id":"chopping_wood","title":"Chopping Wood","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/chopping_wood.jpg?v=72ecbacf","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/chopping_wood.mp4?v=c49468fb"},{"suite":"myosuite","id":"table_tennis","title":"Table Tennis","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/table_tennis.jpg?v=405fa27e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/wall/table_tennis.mp4?v=16f8f308"},{"suite":"dexjoco","id":"fold_glasses","title":"Fold Glasses","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/fold_glasses.jpg?v=6da13f7d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexjoco/wall/fold_glasses.mp4?v=7eeee30d"},{"suite":"dexhand","id":"bicycle_pump_stroke","title":"Bicycle Pump Stroke","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/bicycle_pump_stroke.jpg?v=32049563","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/dexhand/bicycle_pump_stroke.mp4?v=ddf04242"},{"suite":"robowits","id":"stand_pages","title":"Stand Pages","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/stand_pages.jpg?v=a6ce7df0","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/wall/stand_pages.mp4?v=6f3bdd0f"},{"suite":"robopaint","id":"oil_monalisa_i01","title":"Oil · Monalisa","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/oil_monalisa_i01.jpg?v=997fbfa7","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/oil_monalisa_i01.mp4?v=58c9bf49"},{"suite":"robolab","id":"unstack_rubiks_cube","title":"Unstack Rubiks Cube","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/unstack_rubiks_cube.jpg?v=65c18d80","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/wall/unstack_rubiks_cube.mp4?v=582692ea"},{"suite":"embodied-olympics-fun","id":"volleyball_spike","title":"Volleyball Spike","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/volleyball_spike.jpg?v=758e100c","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/wall/volleyball_spike.mp4?v=ea4d3596"},{"suite":"roboarc","id":"lock_picking_02","title":"Lock Picking 02","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/lock_picking_02.jpg?v=eacb8523","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/lock_picking_02.mp4?v=c8dce7a6"},{"suite":"behavior-1k","id":"hanging_pictures","title":"Hanging Pictures","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/hanging_pictures.jpg?v=72632c71","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/hanging_pictures.mp4?v=35d9cce4"},{"suite":"myosuite","id":"die_reorient","title":"Die Reorient","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/die_reorient.jpg?v=061ec48d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/wall/die_reorient.mp4?v=ea1f5dcf"},{"suite":"robowits","id":"hold_cup","title":"Hold Cup","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/hold_cup.jpg?v=996938f4","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/wall/hold_cup.mp4?v=8f2df752"},{"suite":"robopaint","id":"kaishu_yongzibafa","title":"Kaishu · Yongzibafa","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/kaishu_yongzibafa.jpg?v=b2a7c5d9","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robopaint/kaishu_yongzibafa.mp4?v=9a46a408"},{"suite":"embodied-olympics-fun","id":"javelin","title":"Javelin","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/javelin.jpg?v=5df07f48","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboolympics/wall/javelin.mp4?v=2ec9391a"},{"suite":"roboarc","id":"rope_threading_04","title":"Rope Threading 04","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/rope_threading_04.jpg?v=dfba46e0","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/rope_threading_04.mp4?v=7e7d443b"},{"suite":"behavior-1k","id":"carrying_out_garden_furniture","title":"Carrying Out Garden Furniture","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/carrying_out_garden_furniture.jpg?v=bddc2c78","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/carrying_out_garden_furniture.mp4?v=4b5fc93f"},{"suite":"myosuite","id":"pen_twirl","title":"Pen Twirl","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/pen_twirl.jpg?v=73a76e9e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/myosuite/wall/pen_twirl.mp4?v=3f01fe8f"},{"suite":"robowits","id":"raise_platform","title":"Raise Platform","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/raise_platform.jpg?v=e01c4ea2","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robowits/wall/raise_platform.mp4?v=61cd2219"},{"suite":"roboarc","id":"hedgehog_in_cage_05","title":"Hedgehog In Cage 05","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/hedgehog_in_cage_05.jpg?v=b4508cd1","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/hedgehog_in_cage_05.mp4?v=c00ffe6b"},{"suite":"behavior-1k","id":"rearranging_kitchen_furniture","title":"Rearranging Kitchen Furniture","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/rearranging_kitchen_furniture.jpg?v=cc2a6d33","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/rearranging_kitchen_furniture.mp4?v=c75dd86e"},{"suite":"roboarc","id":"fiber_bundling_06","title":"Fiber Bundling 06","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/fiber_bundling_06.jpg?v=09a63e7e","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/fiber_bundling_06.mp4?v=3fb3fe5c"},{"suite":"behavior-1k","id":"make_pizza","title":"Make Pizza","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/make_pizza.jpg?v=8ca854e7","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/wall/make_pizza.mp4?v=bd6a4e61"},{"suite":"roboarc","id":"cast_keyhole_03","title":"Cast Keyhole 03","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/cast_keyhole_03.jpg?v=efd8345d","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/cast_keyhole_03.mp4?v=52a93bf6"},{"suite":"roboarc","id":"horseshoe_ring_02","title":"Horseshoe Ring 02","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/horseshoe_ring_02.jpg?v=85c99b82","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/horseshoe_ring_02.mp4?v=53ba4ad8"},{"suite":"roboarc","id":"slotted_post_ring_03","title":"Slotted Post Ring 03","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/slotted_post_ring_03.jpg?v=7753e058","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/wall/slotted_post_ring_03.mp4?v=660a9ed1"},{"suite":"roboarc","id":"cast_coil_04","title":"Cast Coil 04","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/cast_coil_04.jpg?v=06ff4eca","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/wall/cast_coil_04.mp4?v=975cb63f"},{"suite":"roboarc","id":"blanket_flatten_04","title":"Blanket Flatten 04","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/blanket_flatten_04.jpg?v=4d48926b","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/wall/blanket_flatten_04.mp4?v=164c112f"},{"suite":"roboarc","id":"interlocked_claws_09","title":"Interlocked Claws 09","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/interlocked_claws_09.jpg?v=de324195","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/interlocked_claws_09.mp4?v=e22e5c52"},{"suite":"roboarc","id":"knot_untying_05","title":"Knot Untying 05","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/knot_untying_05.jpg?v=72d00984","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/wall/knot_untying_05.mp4?v=86b5fd2c"},{"suite":"roboarc","id":"rope_threading_07","title":"Rope Threading 07","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/rope_threading_07.jpg?v=7b226e02","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/wall/rope_threading_07.mp4?v=52bafafd"},{"suite":"roboarc","id":"buzz_wire_3d_09","title":"Buzz Wire 3D 09","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/buzz_wire_3d_09.jpg?v=6c0bd3f1","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/wall/buzz_wire_3d_09.mp4?v=65fe533d"},{"suite":"roboarc","id":"lock_picking_04","title":"Lock Picking 04","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/lock_picking_04.jpg?v=722c97e7","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/wall/lock_picking_04.mp4?v=3266a3f9"},{"suite":"roboarc","id":"hedgehog_in_cage_08","title":"Hedgehog In Cage 08","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/hedgehog_in_cage_08.jpg?v=9f343797","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/wall/hedgehog_in_cage_08.mp4?v=5c8e609d"},{"suite":"roboarc","id":"fiber_bundling_09","title":"Fiber Bundling 09","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/fiber_bundling_09.jpg?v=dfa60a29","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/wall/fiber_bundling_09.mp4?v=3e29ffc3"},{"suite":"roboarc","id":"ikea_lack","title":"IKEA Lack","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_lack.jpg?v=9dcee3d2","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_lack.mp4?v=c6c0f2f0"},{"suite":"roboarc","id":"ikea_eket","title":"IKEA Eket","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_eket.jpg?v=a41b4f69","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/wall/ikea_eket.mp4?v=2639c0c1"},{"suite":"roboarc","id":"ikea_svallet","title":"IKEA Svallet","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_svallet.jpg?v=9d89b227","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_svallet.mp4?v=15ed594f"},{"suite":"roboarc","id":"ikea_variera","title":"IKEA Variera","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_variera.jpg?v=1d8db5d6","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/wall/ikea_variera.mp4?v=a01dc48e"},{"suite":"roboarc","id":"ikea_vesken","title":"IKEA Vesken","poster":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/ikea_vesken.jpg?v=f09c5a47","video":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/roboarc/wall/ikea_vesken.mp4?v=439ba775"}],"front":["roboarc","myosuite","embodied-olympics-fun","behavior-1k","dexjoco","robocasa365","dexhand","pandadyn","robopaint","robotwin-2","ambench","robowits","robocasa-gr1","maniguard","yamlab","mujoco-playground","dextoolbench","embodied-safety-exam","autobio"],"back":["humanoidbench","metaworldplus","robodojo","kinder","robolab","vlabench"],"centre":"roboarc/interlocked_claws_05"},"taxmap":{"rows":[{"id":"perception-understanding","name":"Perception & Understanding","description":"Perception, object/state recognition, semantic grounding, spatial understanding.","n":179},{"id":"planning-reasoning","name":"Planning & Reasoning","description":"Task planning, sequencing, tool/affordance reasoning, geometric reasoning.","n":291},{"id":"control-coordination","name":"Control & Coordination","description":"Manipulation/control, contact-rich execution, locomotion control, multi-arm or whole-body coordination.","n":469},{"id":"feedback-adaptation","name":"Feedback & Adaptation","description":"Closed-loop correction, failure recovery, online adaptation, exploration based on feedback.","n":61}],"cols":[{"id":"manipulation","name":"Manipulation","description":"Fixed-base object manipulation, grasping, pushing, insertion, tool use, articulated/deformable object interaction.","n":332},{"id":"locomotion-stability","name":"Locomotion & Stability","description":"Walking, running, balancing, terrain traversal, dynamic stabilization.","n":4},{"id":"navigation-exploration","name":"Navigation & Exploration","description":"Navigation, target search, exploration, spatial memory.","n":40},{"id":"mobile-whole-body-manipulation","name":"Mobile / Whole-body Manipulation","description":"Tasks combining navigation/locomotion with manipulation or requiring coordinated whole-body motion.","n":132},{"id":"interaction-collaboration","name":"Interaction & Collaboration","description":"Human-robot interaction, multi-robot collaboration, handover, cooperative tasks.","n":3}],"cells":{"perception-understanding|manipulation":{"n":112,"suites":8,"ex":[{"s":"robolab","id":"apple_and_yogurt_in_bowl","t":"Apple And Yogurt In Bowl","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/apple_and_yogurt_in_bowl.jpg?v=38f8bac5"},{"s":"robocasa","id":"restock_pantry","t":"Multi-stage · RestockPantry","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/restock_pantry.jpg?v=4b23ba7b"},{"s":"robocasa365","id":"choose_measuring_cup","t":"Composite · ChooseMeasuringCup","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/choose_measuring_cup.jpg?v=d4cd94ec"},{"s":"vlabench","id":"select_billiards_common_sense","t":"Common sense · select_billiards_common_sense","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/select_billiards_common_sense.jpg?v=b86f50fa"}]},"perception-understanding|navigation-exploration":{"n":28,"suites":3,"ex":[{"s":"behavior-1k","id":"boxing_food_after_dinner","t":"Boxing Food After Dinner","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/boxing_food_after_dinner.jpg?v=77629709"},{"s":"molmospaces","id":"navigate_to","t":"RB-Y1 · Navigate to","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/navigate_to.jpg?v=7c95f3fc"},{"s":"kinder","id":"dynamo","t":"Dynamic3D · Dynamo","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/dynamo.jpg?v=df3ff8c4"}]},"perception-understanding|mobile-whole-body-manipulation":{"n":66,"suites":3,"ex":[{"s":"behavior-1k","id":"assembling_gift_baskets","t":"Assembling Gift Baskets","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/assembling_gift_baskets.jpg?v=d99deb25"},{"s":"robocasa365","id":"beverage_sorting","t":"Composite · BeverageSorting","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/beverage_sorting.jpg?v=bbe7eccf"},{"s":"kinder","id":"dynamo","t":"Dynamic3D · Dynamo","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/dynamo.jpg?v=df3ff8c4"}]},"perception-understanding|interaction-collaboration":{"n":1,"suites":1,"ex":[{"s":"robotwin-2","id":"handover_block","t":"Handover block","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/handover_block.jpg?v=e56dc2ae"}]},"planning-reasoning|manipulation":{"n":165,"suites":11,"ex":[{"s":"robolab","id":"apple_and_yogurt_in_bowl","t":"Apple And Yogurt In Bowl","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/apple_and_yogurt_in_bowl.jpg?v=38f8bac5"},{"s":"robocasa","id":"arrange_vegetables","t":"Multi-stage · ArrangeVegetables","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/arrange_vegetables.jpg?v=67364597"},{"s":"robocasa365","id":"bread_and_cheese","t":"Composite · BreadAndCheese","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/bread_and_cheese.jpg?v=ca493423"},{"s":"robocasa-gr1","id":"bottle_to_cabinet_close","t":"GR1 · PnPBottleToCabinetClose","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/bottle_to_cabinet_close.jpg?v=54d1349a"}]},"planning-reasoning|navigation-exploration":{"n":38,"suites":3,"ex":[{"s":"behavior-1k","id":"boxing_food_after_dinner","t":"Boxing Food After Dinner","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/boxing_food_after_dinner.jpg?v=77629709"},{"s":"humanoidbench","id":"manip_door","t":"Manipulation · Door","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/manip_door.jpg?v=992050cf"},{"s":"kinder","id":"dynamo","t":"Dynamic3D · Dynamo","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/dynamo.jpg?v=df3ff8c4"}]},"planning-reasoning|mobile-whole-body-manipulation":{"n":126,"suites":4,"ex":[{"s":"behavior-1k","id":"assembling_gift_baskets","t":"Assembling Gift Baskets","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/assembling_gift_baskets.jpg?v=d99deb25"},{"s":"robocasa365","id":"add_lemon_to_fish","t":"Composite · AddLemonToFish","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/add_lemon_to_fish.jpg?v=4b3f2a2f"},{"s":"humanoidbench","id":"manip_bookshelf_simple","t":"Manipulation · Bookshelf simple","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/manip_bookshelf_simple.jpg?v=388083f5"},{"s":"kinder","id":"constrained_cupboard","t":"Dynamic3D · Constrained cupboard","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/constrained_cupboard.jpg?v=a91f3451"}]},"planning-reasoning|interaction-collaboration":{"n":3,"suites":2,"ex":[{"s":"robotwin-2","id":"handover_block","t":"Handover block","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/handover_block.jpg?v=e56dc2ae"},{"s":"mujoco-playground","id":"aloha_hand_over","t":"Aloha Hand Over","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/aloha_hand_over.jpg?v=20f7a371"}]},"control-coordination|manipulation":{"n":332,"suites":12,"ex":[{"s":"robolab","id":"apple_and_yogurt_in_bowl","t":"Apple And Yogurt In Bowl","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/apple_and_yogurt_in_bowl.jpg?v=38f8bac5"},{"s":"robocasa","id":"arrange_vegetables","t":"Multi-stage · ArrangeVegetables","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa/arrange_vegetables.jpg?v=67364597"},{"s":"robocasa365","id":"close_drawer","t":"Atomic · CloseDrawer","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/close_drawer.jpg?v=c3ae0780"},{"s":"robocasa-gr1","id":"bottle_to_cabinet_close","t":"GR1 · PnPBottleToCabinetClose","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa-gr1/bottle_to_cabinet_close.jpg?v=54d1349a"}]},"control-coordination|locomotion-stability":{"n":4,"suites":1,"ex":[{"s":"humanoidbench","id":"loco_balance_hard","t":"Locomotion · Balance hard","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/loco_balance_hard.jpg?v=71b6721d"}]},"control-coordination|navigation-exploration":{"n":40,"suites":4,"ex":[{"s":"behavior-1k","id":"boxing_food_after_dinner","t":"Boxing Food After Dinner","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/boxing_food_after_dinner.jpg?v=77629709"},{"s":"humanoidbench","id":"manip_door","t":"Manipulation · Door","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/manip_door.jpg?v=992050cf"},{"s":"molmospaces","id":"navigate_to","t":"RB-Y1 · Navigate to","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/navigate_to.jpg?v=7c95f3fc"},{"s":"kinder","id":"dynamo","t":"Dynamic3D · Dynamo","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/dynamo.jpg?v=df3ff8c4"}]},"control-coordination|mobile-whole-body-manipulation":{"n":132,"suites":5,"ex":[{"s":"behavior-1k","id":"assembling_gift_baskets","t":"Assembling Gift Baskets","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/assembling_gift_baskets.jpg?v=d99deb25"},{"s":"robocasa365","id":"add_lemon_to_fish","t":"Composite · AddLemonToFish","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robocasa365/add_lemon_to_fish.jpg?v=4b3f2a2f"},{"s":"humanoidbench","id":"manip_bookshelf_simple","t":"Manipulation · Bookshelf simple","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/manip_bookshelf_simple.jpg?v=388083f5"},{"s":"molmospaces","id":"open_door","t":"RB-Y1 · Open door","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/open_door.jpg?v=a3ca097c"}]},"control-coordination|interaction-collaboration":{"n":3,"suites":2,"ex":[{"s":"robotwin-2","id":"handover_block","t":"Handover block","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/handover_block.jpg?v=e56dc2ae"},{"s":"mujoco-playground","id":"aloha_hand_over","t":"Aloha Hand Over","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/mujoco-playground/aloha_hand_over.jpg?v=20f7a371"}]},"feedback-adaptation|manipulation":{"n":3,"suites":3,"ex":[{"s":"robolab","id":"cubes_and_blocks_in_bin","t":"Cubes And Blocks In Bin","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robolab/cubes_and_blocks_in_bin.jpg?v=d5736401"},{"s":"vlabench","id":"find_unseen_object","t":"Manipulation · find_unseen_object","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/vlabench/find_unseen_object.jpg?v=70fabb80"},{"s":"robotwin-2","id":"put_bottles_dustbin","t":"Put bottles dustbin","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/robotwin-2/put_bottles_dustbin.jpg?v=5da402e1"}]},"feedback-adaptation|locomotion-stability":{"n":2,"suites":1,"ex":[{"s":"humanoidbench","id":"loco_balance_hard","t":"Locomotion · Balance hard","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/humanoidbench/loco_balance_hard.jpg?v=71b6721d"}]},"feedback-adaptation|navigation-exploration":{"n":24,"suites":3,"ex":[{"s":"behavior-1k","id":"boxing_food_after_dinner","t":"Boxing Food After Dinner","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/boxing_food_after_dinner.jpg?v=77629709"},{"s":"molmospaces","id":"navigate_to","t":"RB-Y1 · Navigate to","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/molmospaces/navigate_to.jpg?v=7c95f3fc"},{"s":"kinder","id":"dynamo","t":"Dynamic3D · Dynamo","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/dynamo.jpg?v=df3ff8c4"}]},"feedback-adaptation|mobile-whole-body-manipulation":{"n":55,"suites":2,"ex":[{"s":"behavior-1k","id":"assembling_gift_baskets","t":"Assembling Gift Baskets","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/behavior-1k/assembling_gift_baskets.jpg?v=d99deb25"},{"s":"kinder","id":"dynamo","t":"Dynamic3D · Dynamo","p":"https://huggingface.co/datasets/MLL-Lab/embodied-exam-data/resolve/main/site/kinder/dynamo.jpg?v=df3ff8c4"}]}},"total":469,"per_question":{"1":132,"2":193,"3":94,"4":50},"top_mix":{"ids":["planning-reasoning","control-coordination"],"n":149}},"folders":{"abc-sim":{"tasks":[0,29],"modes":{"privileged":[0,29],"realtime":[0,2],"standard":[0,29]}},"ambench":{"tasks":[2,9],"modes":{"privileged":[2,8],"standard":[2,9]}},"autobio":{"tasks":[0,9],"modes":{"privileged":[0,9],"standard":[0,9]}},"behavior-1k":{"tasks":[71,100],"modes":{"privileged":[71,100],"standard":[71,100]}},"bigym2":{"tasks":[10,13],"modes":{"privileged":[10,13],"realtime":[10,13],"standard":[10,13]}},"dexhand":{"tasks":[23,30],"modes":{"privileged":[23,30],"standard":[23,30]}},"dexjoco":{"tasks":[2,11],"modes":{"privileged":[2,11],"standard":[2,11]}},"dextoolbench":{"tasks":[20,21],"modes":{"privileged":[20,21],"standard":[20,21]}},"embodied-olympics-fun":{"tasks":[56,79],"modes":{"privileged":[56,79],"standard":[56,79],"realtime":[11,21]}},"embodied-olympics-match":{"tasks":[13,14],"modes":{"privileged":[13,14],"realtime":[12,13],"standard":[13,14]}},"embodied-safety-exam":{"tasks":[50,72],"modes":{"standard":[50,71],"realtime":[4,5]}},"embodiedswe":{"tasks":[12,24],"modes":{"privileged":[12,24],"standard":[12,24]}},"humanoidbench":{"tasks":[8,10],"modes":{"privileged":[8,10],"realtime":[8,10],"standard":[8,10]}},"ispatialgym":{"tasks":[8,12],"modes":{"privileged":[8,12],"standard":[8,12]}},"kinder":{"tasks":[4,5],"modes":{"privileged":[4,5],"standard":[4,5]}},"labutopia":{"tasks":[0,9],"modes":{"privileged":[0,9],"standard":[0,9]}},"libero-max":{"tasks":[2,64],"modes":{"privileged":[2,64],"standard":[2,64]}},"maniguard":{"tasks":[7,30],"modes":{"privileged":[7,30],"standard":[7,30]}},"metaworldplus":{"tasks":[2,50],"modes":{"privileged":[2,50],"standard":[2,50]}},"molmospaces":{"tasks":[2,8],"modes":{"privileged":[2,8],"standard":[2,8]}},"mujoco-playground":{"tasks":[7,13],"modes":{"privileged":[7,13],"standard":[7,13],"realtime":[1,1]}},"myosuite":{"tasks":[13,22],"modes":{"privileged":[13,22],"standard":[13,22],"realtime":[0,1]}},"pandadyn":{"tasks":[10,25],"modes":{"privileged":[10,25],"realtime":[8,19],"standard":[10,25]}},"roboarc":{"tasks":[58,106],"modes":{"privileged":[58,106],"standard":[58,106]}},"robocasa":{"tasks":[2,29],"modes":{"privileged":[2,29],"standard":[2,29]}},"robocasa-gr1":{"tasks":[1,24],"modes":{"privileged":[1,24],"standard":[1,24]}},"robocasa365":{"tasks":[5,50],"modes":{"privileged":[5,50],"standard":[5,50],"realtime":[0,1]}},"robodojo":{"tasks":[37,54],"modes":{"privileged":[37,54],"standard":[37,54]}},"roboengineer":{"tasks":[15,30],"modes":{"standard":[15,30]}},"robolab":{"tasks":[6,53],"modes":{"privileged":[6,53],"standard":[6,53]}},"robopaint":{"tasks":[22,43],"modes":{"privileged":[22,43],"standard":[22,43]}},"robotwin-2":{"tasks":[3,47],"modes":{"privileged":[3,47],"standard":[3,47]}},"robowits":{"tasks":[6,15],"modes":{"privileged":[6,15],"standard":[6,15]}},"vlabench":{"tasks":[2,36],"modes":{"privileged":[2,36],"standard":[2,36]}},"yamlab":{"tasks":[2,10],"modes":{"privileged":[2,10],"standard":[2,10]}}},"unranked_runs":[]}