{
  "id": "placement-policy-selection-block16-v2",
  "status": "registered_before_heldout_evaluation",
  "registered_at": "2026-10-16T10:48:11.067368+00:00",
  "question": "Can a learned visual world select a better fixed placement policy than a default policy, measured against held-out native simulator outcomes?",
  "utility_protocol_sha256": "fd40a28117500056d5363cadb8aca246a17c5f94d68bf78f6a28b8b11bb69571",
  "policy_cohort": {
    "arms": [
      "reference-only",
      "ordinary-augmentation",
      "cosmos3-edge-images",
      "cosmos3-nano-images"
    ],
    "training_seed": 202610260,
    "checkpoint_rule": "All four final 3000-step checkpoints from the registered utility experiment, fixed before held-out outcomes; no best-seed or best-epoch selection."
  },
  "cases": [
    {
      "id": "heldout-layout02-00",
      "split": "heldout",
      "position_pair_group": 1,
      "layout_id": 2,
      "scene_seed": 202610281,
      "xy_jitter_m": [
        [
          0.0057129476025190515,
          -0.001590140471625116
        ],
        [
          -0.007769177835857493,
          -0.011220915149745747
        ]
      ]
    },
    {
      "id": "heldout-layout03-00",
      "split": "heldout",
      "position_pair_group": 1,
      "layout_id": 3,
      "scene_seed": 202610283,
      "xy_jitter_m": [
        [
          0.00618779139129403,
          -0.014944232940357209
        ],
        [
          0.005603354334395461,
          0.010381698143923385
        ]
      ]
    },
    {
      "id": "heldout-layout04-00",
      "split": "heldout",
      "position_pair_group": 2,
      "layout_id": 4,
      "scene_seed": 202610285,
      "xy_jitter_m": [
        [
          -0.009399788364165822,
          0.003669167387165894
        ],
        [
          0.0103905456938456,
          -0.014944134354850118
        ]
      ]
    },
    {
      "id": "heldout-layout05-00",
      "split": "heldout",
      "position_pair_group": 2,
      "layout_id": 5,
      "scene_seed": 202610287,
      "xy_jitter_m": [
        [
          -0.013115807264596397,
          -0.0007921298858045971
        ],
        [
          -0.00894814760463169,
          0.004199190924903666
        ]
      ]
    },
    {
      "id": "heldout-layout06-00",
      "split": "heldout",
      "position_pair_group": 3,
      "layout_id": 6,
      "scene_seed": 202610289,
      "xy_jitter_m": [
        [
          -0.000539334141665954,
          0.012809547899495344
        ],
        [
          -0.0014232983030121708,
          0.005560445751295294
        ]
      ]
    },
    {
      "id": "heldout-layout07-00",
      "split": "heldout",
      "position_pair_group": 3,
      "layout_id": 7,
      "scene_seed": 202610291,
      "xy_jitter_m": [
        [
          -0.00048728073978110235,
          -0.0013923294008176216
        ],
        [
          -0.00614181956669029,
          -0.003399284896810035
        ]
      ]
    },
    {
      "id": "heldout-layout14-00",
      "split": "heldout",
      "position_pair_group": 7,
      "layout_id": 14,
      "scene_seed": 202610293,
      "xy_jitter_m": [
        [
          0.010832525973179852,
          -0.0032474437634227964
        ],
        [
          0.011937648676346533,
          0.0057119710663981055
        ]
      ]
    },
    {
      "id": "heldout-layout15-00",
      "split": "heldout",
      "position_pair_group": 7,
      "layout_id": 15,
      "scene_seed": 202610295,
      "xy_jitter_m": [
        [
          -0.014596722793708923,
          -0.0016573619332378112
        ],
        [
          0.002424152841036479,
          -0.006791325549598234
        ]
      ]
    },
    {
      "id": "heldout-layout18-00",
      "split": "heldout",
      "position_pair_group": 9,
      "layout_id": 18,
      "scene_seed": 202610297,
      "xy_jitter_m": [
        [
          0.007488777161673548,
          -0.013378760589206501
        ],
        [
          0.0031087897268869645,
          0.008818017159782258
        ]
      ]
    },
    {
      "id": "heldout-layout19-00",
      "split": "heldout",
      "position_pair_group": 9,
      "layout_id": 19,
      "scene_seed": 202610299,
      "xy_jitter_m": [
        [
          0.005875851005415822,
          0.013178097067496123
        ],
        [
          -0.00974840594894346,
          -0.009088311150062714
        ]
      ]
    },
    {
      "id": "heldout-layout20-00",
      "split": "heldout",
      "position_pair_group": 10,
      "layout_id": 20,
      "scene_seed": 202610301,
      "xy_jitter_m": [
        [
          -0.008276762562727557,
          0.009415035982903823
        ],
        [
          -0.012896231991920706,
          -0.010874214180162557
        ]
      ]
    },
    {
      "id": "heldout-layout21-00",
      "split": "heldout",
      "position_pair_group": 10,
      "layout_id": 21,
      "scene_seed": 202610303,
      "xy_jitter_m": [
        [
          -0.011365138839512495,
          0.0032561235069519454
        ],
        [
          0.012789207485046993,
          0.012189388503094162
        ]
      ]
    }
  ],
  "worlds": [
    "cosmos3-edge",
    "cosmos3-nano"
  ],
  "generation_seeds": [
    202610310,
    202610311
  ],
  "horizon_steps": 80,
  "control_hz": 5,
  "interface": {
    "policy_inputs": [
      "current RGB",
      "current robot qpos"
    ],
    "learned_proprioception": "Independent no-contact actuator qpos initialized only from the reset. It omits grasp-load and contact feedback.",
    "world_action_adapter": "Hold most recent RGB for 16 sequential 5Hz policy decisions using evolving isolated actuator qpos. Condition native chunk_size=16 on those executed actuator poses; consume all 16 generated frames and feed the final frame back for the next block. Five blocks, 80 control steps; visual updates at 0.3125Hz.",
    "reference_access": "Reference reset supplies initial RGB and robot state only. No subsequent reference stepping, observations, object states or future trajectories enter learned episodes.",
    "configuration": "Pinned Cosmos3 checkpoints and Diffusers revision from the qualifying BC pilot; 30 steps, guidance 1, flow shift 10, 320x256, 5 fps. No changes after evaluation starts.",
    "visual_block_steps": 16
  },
  "reference_truth": "Run all four fixed checkpoints on all twelve cases with the identical held-RGB block16 schedule and measured reference qpos. Native placement predicate true in all final five control steps. Do not substitute full-rate visual utility outcomes; show those separately as an observation-rate sensitivity analysis.",
  "world_outcome_evaluator": {
    "type": "Blinded automated visual judgment, not physical truth",
    "labels": [
      "S",
      "F",
      "U"
    ],
    "rubric": "Same carrot visibly released and supported by the plate at the end; occlusion, disappearance, duplication or uncertain contact = U. Exact frozen evaluator configuration and frame hashes accompany each judgment.",
    "qualification": "Calibrate on native successful/unsuccessful rollouts and controlled visual negatives; audit generated clips blinded to model and automatic label before comparing labels. Independent human validation remains pending. If generated judgments cannot support reliable discrimination, publish abstention and audit evidence instead of a model recommendation."
  },
  "selection": {
    "rule": "For each world, choose the policy with highest fraction of S across all 12 cases and both generation seeds; count U and generation/evaluator failures as zero in this conservative operational rule.",
    "tie_break": "First policy in fixed cohort order. Report all ties.",
    "default_policy": "reference-only",
    "regret": "Best native reference success among the four minus native success of the selected policy, equal-weight six position-pair groups.",
    "unknown_sensitivity": "Also report each policy success interval with all U/failures treated as F versus S. Do not present the conservative lower bound as ground-truth failure.",
    "metrics": [
      "All policy native success rates",
      "All world-estimated S/F/U rates",
      "Selection regret",
      "Selected-policy improvement over reference-only",
      "Kendall tau-b on policy means; undefined when all tied",
      "False success among native reference failures",
      "Unknown and execution-failure rates",
      "Generation latency/memory/cost"
    ],
    "uncertainty": "Paired cluster bootstrap across six position-pair groups and two generation seeds, 2000 draws seed 202610320. Keep orientations nested within group. This does not represent independent training-seed uncertainty.",
    "interpretability_gate": "If native policy outcomes are tied or all policies fail, disclose that selection utility is unidentifiable on this cohort; never call zero regret evidence of a useful selector."
  },
  "visual_dependence_control": "All four policies on twelve identical resets with RGB fixed to initial image and live measured reference qpos, compared with block16 and full-rate reference outcomes.",
  "limits": [
    "One task, one embodiment, small Rather-trained recurrent policies; not a broad pretrained VLA claim.",
    "Learned visual worlds plus a no-contact actuator approximation are not full contact-dynamics simulators.",
    "80-step Rather extension, not the official Simpler score.",
    "Independent humans and physical robots unavailable; validation packages prepared.",
    "Visual feedback is 0.3125Hz, commands 5Hz. This is a slow visual feedback interface, not a 5Hz observation world or real-time robotics deployment."
  ],
  "qualification_gate": "Block16 development reference 2/2; four generated scenes remain recognizable at beginning/middle/end, with object identity failures retained. Closed-reference actuator lifetime qualification must also pass before learned episodes. Exact four training checkpoints verified.",
  "supersedes_unexecuted_protocol": {
    "sha256": "00cb643b62dc1d768caac425f81ba22dee318d115347f1dddf60055dd44b7e73",
    "reason": "One-frame feedback failed development visual qualification. Block16 passed the predefined scene-collapse gate; no held-out selection outputs observed."
  }
}