[Real-world active learning] PR 7: human-gated task rebuild on the stock loop

predicators/envs/pybullet_env.py

@@ -72,13 +72,22 @@ class ActionExecutor(Protocol):
     """Drives real hardware from a simulated env's rollout.
 
     The port an env exposes for "something outside is executing what I
-    just simulated". Declared here, next to the two calls, and
+    just simulated". Declared here, next to the three calls, and
     implemented elsewhere -- ``pybullet_helpers.real_robot_executor`` --
     so this module never imports anything that knows about a robot.
 
     An env with no executor attached is pure simulation.
     """
 
+    def tasks_for(self, train_or_test: str) -> Optional[List[EnvironmentTask]]:
+        """Tasks to use for this split, or None to keep the env's own.
+
+        Asked *every* time the tasks are requested, because for a real
+        environment "give me the train task" genuinely means "look at
+        the world". An executor that has nothing new to say returns None
+        and the env's own generated (and cached) tasks stand.
+        """
+
     def after_reset(self, train_or_test: str, task_idx: int,
                     obs: Observation) -> None:
         """The env has been reset to a task's initial state."""
@@ -761,6 +770,44 @@ def attach_executor(self, executor: ActionExecutor) -> None:
         """
         self._executor = executor
 
+    def get_train_tasks(self) -> List[EnvironmentTask]:
+        """The train tasks, letting an executor supply them instead.
+
+        ``BaseEnv`` caches these, which is right for a simulated env
+        whose tasks are generated once. For a real one it is wrong:
+        every episode faces a physically different scene, and the task
+        has to be rebuilt from what is actually there. The executor is
+        asked first; see ``ActionExecutor.tasks_for``.
+        """
+        self._maybe_replace_tasks("train")
+        return super().get_train_tasks()
+
+    def get_test_tasks(self) -> List[EnvironmentTask]:
+        """The test tasks, letting an executor supply them instead."""
+        self._maybe_replace_tasks("test")
+        return super().get_test_tasks()
+
+    def _maybe_replace_tasks(self, train_or_test: str) -> None:
+        """Overwrite the cached tasks for a split if the executor has new
+        ones."""
+        if self._executor is None:
+            return
+        fresh = self._executor.tasks_for(train_or_test)
+        if fresh is None:
+            return
+        cached = (self._train_tasks
+                  if train_or_test == "train" else self._test_tasks)
+        if cached and len(fresh) < len(cached):
+            # Keep the list length: task *indices* have already been handed out
+            # against the old length, so shrinking it would turn a live index
+            # into an IndexError. A real environment perceives one world, so
+            # every index legitimately names the same freshly-perceived scene.
+            fresh = [fresh[0]] * len(cached)
+        if train_or_test == "train":
+            self._train_tasks = fresh
+        else:
+            self._test_tasks = fresh
+
     # ── State Write (State → PyBullet) ──────────────────────────
 
     def sync_to_state(self, state: State) -> None:

predicators/pybullet_helpers/real_robot_bridge.py

@@ -133,6 +133,20 @@ def reset_arm(robot: "RealRobot", joints: Sequence[float]) -> Sequence[float]:
     return reply.joints
 
 
+def reset_env(robot: "RealRobot",
+              joints: Optional[Sequence[float]] = None) -> Any:
+    """Home the arm, wait for a human to rearrange the scene, then look.
+    Blocking, and deliberately unbounded.
+
+    Returns the observation captured *after* the human confirmed.
+    """
+    # pylint: disable=import-outside-toplevel,import-error
+    from babyrobot.realrobot.messages import ResetEnvRequest
+    request = ResetEnvRequest(
+        joints=tuple(joints) if joints is not None else None)
+    return robot.reset_env(request)
+
+
 def execute_chunks(robot: "RealRobot",
                    chunks: Sequence[Sequence[Action]],
                    layout: GripperJointLayout,
@@ -143,7 +157,7 @@ def execute_chunks(robot: "RealRobot",
 
     One chunk is one unit of "execute this, then optionally look": with
     ``observe`` the reply carries one observation per chunk, which is
-    how the executor gets a look at the bench per option. With
+    how the executor gets a look at the scene per option. With
     ``observe=False`` nothing comes back and the caller's world state
     stays the sim's prediction.
 

predicators/pybullet_helpers/real_robot_executor.py

@@ -2,7 +2,7 @@
 
 predicators rolls each option out in simulation exactly as it always has. This
 module ships the resulting joint trajectory to the real arm, looks at the
-bench, and writes what it saw back into the simulated **twin**. The env stays
+scene, and writes what it saw back into the simulated **twin**. The env stays
 pure simulation and never learns that a robot exists: it exposes one optional
 collaborator (``PyBulletEnv.attach_executor``) and calls it after each reset
 and each step.
@@ -25,7 +25,7 @@
 from predicators.envs.base_env import BaseEnv
 from predicators.envs.pybullet_env import PyBulletEnv
 from predicators.pybullet_helpers.real_robot_bridge import execute_chunks, \
-    make_real_robot, reset_arm
+    make_real_robot, reset_arm, reset_env
 from predicators.settings import CFG
 from predicators.structs import Action, EnvironmentTask, Observation, State
 
@@ -98,14 +98,14 @@ def __init__(self, env: PyBulletEnv, divergence_atol: float) -> None:
         self._env = env
         self._divergence_atol = divergence_atol
         # Largest per-object position disagreement between the twin and the
-        # bench at the last look, in metres; None before the first look.
+        # real scene at the last look, in metres; None before the first look.
         self.last_divergence: Optional[float] = None
 
     def absorb(self, observation: Any) -> Observation:
         """Write what the cameras saw into the twin; return its new reading.
 
         Divergence is measured against the twin's *pre-sync* state --
-        the sim's prediction of where the bench would be -- because that
+        the sim's prediction of where the scene would be -- because that
         is the comparison that says reality went somewhere the model did
         not. ``_set_state``'s own reconstruction check cannot answer
         this: it round-trips the state it was asked to write and
@@ -121,7 +121,7 @@ def absorb(self, observation: Any) -> Observation:
         if self.last_divergence is not None and \
                 self.last_divergence > self._divergence_atol:
             logging.warning(
-                "real robot: the bench is %.3f m from where the twin "
+                "real robot: the scene is %.3f m from where the twin "
                 "predicted (tolerance %.3f m); the twin is being corrected, "
                 "but the current plan was made against the prediction",
                 self.last_divergence, self._divergence_atol)
@@ -145,7 +145,8 @@ def __init__(self,
                  robot: Any,
                  observe_at_boundaries: bool = True,
                  settle_s: float = 0.0,
-                 divergence_atol: float = 0.02) -> None:
+                 divergence_atol: float = 0.02,
+                 human_reset: bool = True) -> None:
         missing = sorted(name for name in _REQUIRED_HOOKS
                          if not callable(getattr(env, name, None)))
         if missing:
@@ -157,35 +158,98 @@ def __init__(self,
         if observe_at_boundaries and not getattr(robot, "has_perception",
                                                  False):
             raise ValueError(
-                "asked to look at the bench between options, but the robot "
+                "asked to look at the scene between options, but the robot "
                 "has no perception configured; set real_robot_perception, or "
                 "turn the option-boundary look off for a blind open-loop run")
+        if human_reset and not getattr(robot, "has_perception", False):
+            raise ValueError(
+                "human resets rebuild each episode's task from the scene, so "
+                "the robot needs perception; set real_robot_perception, or "
+                "turn real_robot_human_reset off to keep the captured scene")
         self._env = env
         self._robot = robot
         self._observe = observe_at_boundaries
         self._settle_s = settle_s
+        self._human_reset = human_reset
         self._buffer = OptionBoundaryBuffer()
         self._corrector = TwinCorrector(env, divergence_atol)
+        # The scene has to be arranged before the first episode, so a reset is
+        # owed from the start. Set again by every reset, i.e. once per episode.
+        self._reset_pending = True
+        # The twin's home arm configuration, captured now: an executor is
+        # attached right after the env is built, so the simulated arm is still
+        # at home. A scene reset has to send the real arm somewhere before the
+        # human reaches in, and this is that somewhere .
+        self._home_arm = self._home_arm_joints(env.get_observation())
+        # How many times the scene has been (re)perceived for a task. Read by
+        # tests and worth logging on a hardware session.
+        self.resets_done = 0
 
     @property
     def last_divergence(self) -> Optional[float]:
-        """How far the bench was from the twin at the last look."""
+        """How far the scene was from the twin at the last look."""
         return self._corrector.last_divergence
 
     # -- the ActionExecutor port -------------------------------------------
+    def tasks_for(self, train_or_test: str) -> Optional[List[EnvironmentTask]]:
+        """Rebuild this split's task from the scene, resetting it first.
+
+        **Why this happens here and not in ``reset``.** The task has to
+        be rebuilt before anything consumes it, and the two callers
+        consume it differently:
+
+        * *Evaluation* solves at reset. ``_solve_task``
+          (``main.py:774``) calls ``cogman.reset(env_task)`` with no
+          override policy, so ``_reset_policy`` runs
+          ``approach.solve(task)`` -- before ``env.reset`` is ever
+          called. A human reset performed in ``env.reset`` would
+          therefore plan against a scene that no longer exists.
+        * *Exploration* does not solve at reset: the online loop sets
+          an override policy first (``main.py:551``), so
+          ``_reset_policy`` takes that branch. The task still matters,
+          because it is what ``env.reset`` initializes the simulated
+          twin from -- a stale one starts every episode from the
+          captured scene rather than the one just arranged.
+
+        For a real environment "give me the train task" honestly means
+        "look at the scene", so that is what it does.
+
+        Returns None -- leaving the env's captured-scene task alone --
+        when no reset is owed, or when human resets are off. The latter
+        is what keeps a fixed-plan replay reproducible.
+        """
+        if not self._human_reset or not self._reset_pending:
+            return None
+        # Homes the arm out of the way, blocks until the human confirms, then
+        # perceives.
+        observation = reset_env(self._robot, self._home_arm)
+        self._reset_pending = False
+        self.resets_done += 1
+        logging.info(
+            "real robot: scene reset #%d; rebuilding the %s task "
+            "from what the cameras see", self.resets_done, train_or_test)
+        domain = cast(_DomainHooks, self._env)
+        # One task, because a physical scene is one scene. The env keeps the
+        # list length stable, so task indices already handed out stay valid.
+        return [domain.task_from_observation(observation, train_or_test)]
+
     def after_reset(self, train_or_test: str, task_idx: int,
                     obs: Observation) -> None:
         """Home the real arm to wherever the twin just reset to.
 
         The twin is reset first because its home joint configuration is
-        what the option trajectories are planned from: the arm has to
-        start where the first option's streamed waypoints begin, or the
-        drift guard trips on the opening move.
+        what the option trajectories are planned from.
+
+        Both train and test splits execute. Real mode is a property of
+        an executor being attached, not of the split.
 
-        Both splits execute. Real mode is a property of an executor
-        being attached, not of the split.
+        This is also where the next reset is owed from: an episode has
+        just begun, so the *following* task request has to face a
+        freshly arranged scene. Marking it here rather than at the end
+        of an episode is what makes it exactly one prompt per episode.
         """
         del train_or_test, task_idx  # every episode homes the same way
+        self._reset_pending = True
         lost = self._buffer.discard()
         if lost:
             # The previous episode ended mid-option (step limit, or an
@@ -267,6 +331,7 @@ def attach_real_robot(env: BaseEnv,
         robot,
         observe_at_boundaries=CFG.real_robot_observe_at_option_boundary,
         settle_s=CFG.real_robot_settle_s,
-        divergence_atol=CFG.real_robot_divergence_atol)
+        divergence_atol=CFG.real_robot_divergence_atol,
+        human_reset=CFG.real_robot_human_reset)
     env.attach_executor(executor)
     return executor

predicators/settings.py

@@ -583,16 +583,21 @@ class GlobalSettings:
     # cameraless, but it always reports the captured layout), or "none" (no
     # cameras at all, blind open-loop run).
     real_robot_perception = "zed"
-    # Look at the bench at each option boundary and correct the twin from what
+    # Look at the scene at each option boundary and correct the twin from what
     # was seen. This is the point of running on real hardware -- the learner
     # sees perceived transitions rather than the simulator's guesses.
     real_robot_observe_at_option_boundary = True
     # Dwell before a capture, so the dominoes come to rest after the motion.
     real_robot_settle_s = 0.5
-    # How far (metres) the bench may be from where the twin predicted before
+    # How far (metres) the scene may be from where the twin predicted before
     # the disagreement is worth logging.
     real_robot_divergence_atol = 0.02
-    # --- real-world domino bench (pybullet_domino_real env) ------------------
+    # Between episodes, home the arm and wait for a human to rearrange the
+    # scene, then rebuild that episode's task from what the cameras then see.
+    # False keeps the captured scene, which is what a fixed-plan
+    # replay wants (rebuilding would change the objects the plan named).
+    real_robot_human_reset = True
+    # --- real-world domino scene (pybullet_domino_real env) ------------------
     # The reconstructed-scene JSON (robot_base frame) the pybullet_domino_real
     # env builds its single train/test task from; the env sizes its domino
     # component from this scene's role counts.
@@ -604,7 +609,7 @@ class GlobalSettings:
     # an explicit 'role' field per domino.
     domino_real_start_id = 6
     domino_real_target_id = 5
-    # Real-bench geometry.
+    # Real-scene geometry.
     domino_real_table_z = -0.041  # real table height in the robot base frame
     domino_real_robot_init_tilt = np.pi
     domino_real_robot_init_wrist = 0.0

scripts/configs/predicatorv3/exp_domino_real.yaml

@@ -1,11 +1,12 @@
-# Thin launcher: real-world domino testing through the STOCK Predicators
-# pipeline (replaces real_skills/run_online_learning.py's bespoke lifecycle,
-# which bypassed setup_environment and hand-transcribed ~70 flags).
+# Thin launcher: real-world domino testing through the Predicators
+# pipeline.
 # Usage: python scripts/local/launch_simp.py -c predicatorv3/exp_domino_real.yaml
 # With real_robot_execute (default off, i.e. pure sim)
 # a RealRobotExecutor is attached and its rollouts drive the real Franka --
-# BOTH splits, so exploration episodes execute like evaluation ones, and the
-# bench is re-perceived between options to correct the simulated twin.
+# BOTH train and testsplits, so exploration episodes execute like evaluation ones, and the
+# scene is re-perceived between skills to correct the simulated twin.
+# Between episodes a human rearranges the dominoes and the episode's task is
+# rebuilt from what the cameras see (real_robot_human_reset).
 # Env definition lives in envs/all.yaml (domino_real) and the approach in
 # approaches/all.yaml; this file only un-skips the env + approach it runs and
 # sets the agent-only ENVS overrides.
@@ -14,6 +15,9 @@ includes:
   - common.yaml
   - envs/all.yaml
   - approaches/all.yaml
+FLAGS:
+  num_online_learning_cycles: 2
+NUM_SEEDS: 1
 # Agent-only env overrides: deep-merged on top of envs/all.yaml. These
 # excluded_predicates are dropped for "ours" runs (matches exp_domino.yaml;
 # oracle.yaml would keep them in).