{
  "status": "completed_post_publication_correction_analysis",
  "created_at": "2026-10-16T09:35:44.788100+00:00",
  "study_id": "robotics-grasp-outcome-v1",
  "original_protocol": {
    "url": "https://rather-media-143952026851-us-west-2.s3.us-west-2.amazonaws.com/public/studies/assets/3c94037b8322ef0769fd5a3874b11c97709e44b25c87f3d06a51b0babb7bbe7e/study.json",
    "sha256": "3c94037b8322ef0769fd5a3874b11c97709e44b25c87f3d06a51b0babb7bbe7e"
  },
  "original_manifest_sha256": "e2ae02c5a0fdaef7f74ea9c76aa29d369fdf95b0ca1e25c586085676ecb88021",
  "attempts_sha256": "8bd516823725aee9cd10e01fe9dee6f07c7ea02034950221d2a825530e0aacfa",
  "layout_audit_sha256": "799606e75d9017d73e991b14f72749931b82d71db45b7ce544fdb38b7c8f4011",
  "finding": "Eight scene seeds cover seven native layouts and six position-pair groups. Seeds 202610154 and 202610157 have identical initial images but their full rollouts are not byte-identical.",
  "selected_correction": "position_pair",
  "method": "Keep all attempts, failures, unknowns and scores. Weight the six position-pair group means equally, nesting orientation and repeated seeds within each group. Recompute the same 2000-draw percentile bootstrap with seed 20261015. Native-layout grouping is a disclosed sensitivity analysis.",
  "limits": [
    "Post-publication correction, not a preregistered analysis.",
    "Six groups are a small sample; intervals are descriptive and do not measure evaluator bias or real-world generalization.",
    "Changing grouping also changes the weighting and therefore may change point estimates. No generated outputs or per-run scores were changed."
  ],
  "analyses": {
    "original_scene_seed": {
      "group_count": 8,
      "members": {
        "scene-202610150": [
          "scene-202610150-center-close",
          "scene-202610150-left-miss",
          "scene-202610150-right-miss",
          "scene-202610150-center-open"
        ],
        "scene-202610151": [
          "scene-202610151-center-close",
          "scene-202610151-left-miss",
          "scene-202610151-right-miss",
          "scene-202610151-center-open"
        ],
        "scene-202610152": [
          "scene-202610152-center-close",
          "scene-202610152-left-miss",
          "scene-202610152-right-miss",
          "scene-202610152-center-open"
        ],
        "scene-202610153": [
          "scene-202610153-center-close",
          "scene-202610153-left-miss",
          "scene-202610153-right-miss",
          "scene-202610153-center-open"
        ],
        "scene-202610154": [
          "scene-202610154-center-close",
          "scene-202610154-left-miss",
          "scene-202610154-right-miss",
          "scene-202610154-center-open"
        ],
        "scene-202610155": [
          "scene-202610155-center-close",
          "scene-202610155-left-miss",
          "scene-202610155-right-miss",
          "scene-202610155-center-open"
        ],
        "scene-202610156": [
          "scene-202610156-center-close",
          "scene-202610156-left-miss",
          "scene-202610156-right-miss",
          "scene-202610156-center-open"
        ],
        "scene-202610157": [
          "scene-202610157-center-close",
          "scene-202610157-left-miss",
          "scene-202610157-right-miss",
          "scene-202610157-center-open"
        ]
      },
      "summaries": [
        {
          "model_id": "cosmos3-edge-robot-action",
          "attempted": 96,
          "succeeded": 96,
          "failed": 0,
          "metrics": {
            "outcome-agreement-percent": {
              "value": 74.62121212121212,
              "ci95": [
                53.125,
                95.45454545454545
              ],
              "observations": 95,
              "independent_groups": 8
            },
            "visible-outcome-percent": {
              "value": 98.95833333333333,
              "ci95": [
                96.875,
                100.0
              ],
              "observations": 96,
              "independent_groups": 8
            },
            "false-success-percent": {
              "value": 31.25,
              "ci95": [
                8.333333333333334,
                54.16666666666667
              ],
              "observations": 45,
              "independent_groups": 8
            },
            "missed-success-percent": {
              "value": 1.8518518518518519,
              "ci95": [
                0.0,
                5.555555555555555
              ],
              "observations": 50,
              "independent_groups": 6
            },
            "generation-seconds": {
              "value": 8.17035222729707,
              "ci95": [
                8.112855796391766,
                8.23648085431584
              ],
              "observations": 96,
              "independent_groups": 8
            }
          },
          "total_duration_seconds": 885.3157466771081
        },
        {
          "model_id": "cosmos3-nano-robot-action",
          "attempted": 96,
          "succeeded": 96,
          "failed": 0,
          "metrics": {
            "outcome-agreement-percent": {
              "value": 77.42424242424242,
              "ci95": [
                55.890151515151516,
                96.5909090909091
              ],
              "observations": 89,
              "independent_groups": 8
            },
            "visible-outcome-percent": {
              "value": 92.70833333333334,
              "ci95": [
                89.58333333333333,
                95.83333333333334
              ],
              "observations": 96,
              "independent_groups": 8
            },
            "false-success-percent": {
              "value": 25.625,
              "ci95": [
                0.0,
                50.625
              ],
              "observations": 41,
              "independent_groups": 8
            },
            "missed-success-percent": {
              "value": 1.8518518518518519,
              "ci95": [
                0.0,
                5.555555555555555
              ],
              "observations": 48,
              "independent_groups": 6
            },
            "generation-seconds": {
              "value": 9.973734985445239,
              "ci95": [
                9.92697539170816,
                10.025682462689778
              ],
              "observations": 96,
              "independent_groups": 8
            }
          },
          "total_duration_seconds": 1055.8536805948243
        },
        {
          "model_id": "frozen-image",
          "attempted": 96,
          "succeeded": 96,
          "failed": 0,
          "metrics": {
            "outcome-agreement-percent": {
              "value": 46.875,
              "ci95": [
                25.0,
                68.75
              ],
              "observations": 96,
              "independent_groups": 8
            },
            "visible-outcome-percent": {
              "value": 100.0,
              "ci95": [
                100.0,
                100.0
              ],
              "observations": 96,
              "independent_groups": 8
            },
            "false-success-percent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "observations": 45,
              "independent_groups": 8
            },
            "missed-success-percent": {
              "value": 100.0,
              "ci95": [
                100.0,
                100.0
              ],
              "observations": 51,
              "independent_groups": 6
            },
            "generation-seconds": {
              "value": null,
              "ci95": null,
              "observations": 0,
              "independent_groups": 0
            }
          },
          "total_duration_seconds": 0.0
        }
      ],
      "comparisons": [
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "outcome-agreement-percent",
          "difference_left_minus_right": {
            "value": -0.09469696969696972,
            "ci95": [
              -3.409090909090909,
              3.125
            ],
            "observations": 88,
            "independent_groups": 8
          },
          "excluded_pairs": 8
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "visible-outcome-percent",
          "difference_left_minus_right": {
            "value": 6.25,
            "ci95": [
              2.0833333333333335,
              10.416666666666668
            ],
            "observations": 96,
            "independent_groups": 8
          },
          "excluded_pairs": 0
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "false-success-percent",
          "difference_left_minus_right": {
            "value": 0.0,
            "ci95": [
              0.0,
              0.0
            ],
            "observations": 41,
            "independent_groups": 8
          },
          "excluded_pairs": 55
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "missed-success-percent",
          "difference_left_minus_right": {
            "value": 0.0,
            "ci95": [
              -5.555555555555555,
              5.555555555555555
            ],
            "observations": 47,
            "independent_groups": 6
          },
          "excluded_pairs": 49
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "generation-seconds",
          "difference_left_minus_right": {
            "value": -1.8033827581481698,
            "ci95": [
              -1.8819754328287672,
              -1.721414551109774
            ],
            "observations": 96,
            "independent_groups": 8
          },
          "excluded_pairs": 0
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "outcome-agreement-percent",
          "difference_left_minus_right": {
            "value": 27.178030303030305,
            "ci95": [
              -18.56060606060606,
              68.75
            ],
            "observations": 95,
            "independent_groups": 8
          },
          "excluded_pairs": 1
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "visible-outcome-percent",
          "difference_left_minus_right": {
            "value": -1.0416666666666667,
            "ci95": [
              -3.125,
              0.0
            ],
            "observations": 96,
            "independent_groups": 8
          },
          "excluded_pairs": 0
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "false-success-percent",
          "difference_left_minus_right": {
            "value": 31.25,
            "ci95": [
              8.333333333333334,
              54.16666666666667
            ],
            "observations": 45,
            "independent_groups": 8
          },
          "excluded_pairs": 51
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "missed-success-percent",
          "difference_left_minus_right": {
            "value": -98.14814814814815,
            "ci95": [
              -100.0,
              -94.44444444444444
            ],
            "observations": 50,
            "independent_groups": 6
          },
          "excluded_pairs": 46
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "generation-seconds",
          "difference_left_minus_right": {
            "value": null,
            "ci95": null,
            "observations": 0,
            "independent_groups": 0
          },
          "excluded_pairs": 96
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "outcome-agreement-percent",
          "difference_left_minus_right": {
            "value": 31.117424242424246,
            "ci95": [
              -12.007575757575756,
              70.83333333333333
            ],
            "observations": 89,
            "independent_groups": 8
          },
          "excluded_pairs": 7
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "visible-outcome-percent",
          "difference_left_minus_right": {
            "value": -7.291666666666667,
            "ci95": [
              -10.416666666666668,
              -4.166666666666667
            ],
            "observations": 96,
            "independent_groups": 8
          },
          "excluded_pairs": 0
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "false-success-percent",
          "difference_left_minus_right": {
            "value": 25.625,
            "ci95": [
              0.0,
              50.625
            ],
            "observations": 41,
            "independent_groups": 8
          },
          "excluded_pairs": 55
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "missed-success-percent",
          "difference_left_minus_right": {
            "value": -98.14814814814815,
            "ci95": [
              -100.0,
              -94.44444444444444
            ],
            "observations": 48,
            "independent_groups": 6
          },
          "excluded_pairs": 48
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "generation-seconds",
          "difference_left_minus_right": {
            "value": null,
            "ci95": null,
            "observations": 0,
            "independent_groups": 0
          },
          "excluded_pairs": 96
        }
      ]
    },
    "native_layout": {
      "group_count": 7,
      "members": {
        "layout-0": [
          "scene-202610150-center-close",
          "scene-202610150-left-miss",
          "scene-202610150-right-miss",
          "scene-202610150-center-open"
        ],
        "layout-9": [
          "scene-202610151-center-close",
          "scene-202610151-left-miss",
          "scene-202610151-right-miss",
          "scene-202610151-center-open"
        ],
        "layout-22": [
          "scene-202610152-center-close",
          "scene-202610152-left-miss",
          "scene-202610152-right-miss",
          "scene-202610152-center-open"
        ],
        "layout-10": [
          "scene-202610153-center-close",
          "scene-202610153-left-miss",
          "scene-202610153-right-miss",
          "scene-202610153-center-open"
        ],
        "layout-8": [
          "scene-202610154-center-close",
          "scene-202610154-left-miss",
          "scene-202610154-right-miss",
          "scene-202610154-center-open",
          "scene-202610157-center-close",
          "scene-202610157-left-miss",
          "scene-202610157-right-miss",
          "scene-202610157-center-open"
        ],
        "layout-16": [
          "scene-202610155-center-close",
          "scene-202610155-left-miss",
          "scene-202610155-right-miss",
          "scene-202610155-center-open"
        ],
        "layout-13": [
          "scene-202610156-center-close",
          "scene-202610156-left-miss",
          "scene-202610156-right-miss",
          "scene-202610156-center-open"
        ]
      },
      "summaries": [
        {
          "model_id": "cosmos3-edge-robot-action",
          "attempted": 96,
          "succeeded": 96,
          "failed": 0,
          "metrics": {
            "outcome-agreement-percent": {
              "value": 70.99567099567099,
              "ci95": [
                46.42857142857143,
                94.8051948051948
              ],
              "observations": 95,
              "independent_groups": 7
            },
            "visible-outcome-percent": {
              "value": 98.80952380952381,
              "ci95": [
                96.42857142857143,
                100.0
              ],
              "observations": 96,
              "independent_groups": 7
            },
            "false-success-percent": {
              "value": 35.714285714285715,
              "ci95": [
                9.523809523809524,
                60.714285714285715
              ],
              "observations": 45,
              "independent_groups": 7
            },
            "missed-success-percent": {
              "value": 2.2222222222222223,
              "ci95": [
                0.0,
                6.666666666666666
              ],
              "observations": 50,
              "independent_groups": 5
            },
            "generation-seconds": {
              "value": 8.177446260527221,
              "ci95": [
                8.108550748532815,
                8.248842683543142
              ],
              "observations": 96,
              "independent_groups": 7
            }
          },
          "total_duration_seconds": 885.3157466771081
        },
        {
          "model_id": "cosmos3-nano-robot-action",
          "attempted": 96,
          "succeeded": 96,
          "failed": 0,
          "metrics": {
            "outcome-agreement-percent": {
              "value": 74.1991341991342,
              "ci95": [
                49.285714285714285,
                96.1038961038961
              ],
              "observations": 89,
              "independent_groups": 7
            },
            "visible-outcome-percent": {
              "value": 92.85714285714286,
              "ci95": [
                89.28571428571429,
                96.42857142857143
              ],
              "observations": 96,
              "independent_groups": 7
            },
            "false-success-percent": {
              "value": 29.285714285714285,
              "ci95": [
                8.571428571428571,
                57.142857142857146
              ],
              "observations": 41,
              "independent_groups": 7
            },
            "missed-success-percent": {
              "value": 2.2222222222222223,
              "ci95": [
                0.0,
                6.666666666666666
              ],
              "observations": 48,
              "independent_groups": 5
            },
            "generation-seconds": {
              "value": 9.98224765992546,
              "ci95": [
                9.936602192731309,
                10.035442991548084
              ],
              "observations": 96,
              "independent_groups": 7
            }
          },
          "total_duration_seconds": 1055.8536805948243
        },
        {
          "model_id": "frozen-image",
          "attempted": 96,
          "succeeded": 96,
          "failed": 0,
          "metrics": {
            "outcome-agreement-percent": {
              "value": 50.0,
              "ci95": [
                28.571428571428573,
                75.0
              ],
              "observations": 96,
              "independent_groups": 7
            },
            "visible-outcome-percent": {
              "value": 100.0,
              "ci95": [
                100.0,
                100.0
              ],
              "observations": 96,
              "independent_groups": 7
            },
            "false-success-percent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "observations": 45,
              "independent_groups": 7
            },
            "missed-success-percent": {
              "value": 100.0,
              "ci95": [
                100.0,
                100.0
              ],
              "observations": 51,
              "independent_groups": 5
            },
            "generation-seconds": {
              "value": null,
              "ci95": null,
              "observations": 0,
              "independent_groups": 0
            }
          },
          "total_duration_seconds": 0.0
        }
      ],
      "comparisons": [
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "outcome-agreement-percent",
          "difference_left_minus_right": {
            "value": -0.10822510822510825,
            "ci95": [
              -3.8961038961038965,
              3.5714285714285716
            ],
            "observations": 88,
            "independent_groups": 7
          },
          "excluded_pairs": 8
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "visible-outcome-percent",
          "difference_left_minus_right": {
            "value": 5.9523809523809526,
            "ci95": [
              2.380952380952381,
              10.714285714285715
            ],
            "observations": 96,
            "independent_groups": 7
          },
          "excluded_pairs": 0
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "false-success-percent",
          "difference_left_minus_right": {
            "value": 0.0,
            "ci95": [
              0.0,
              0.0
            ],
            "observations": 41,
            "independent_groups": 7
          },
          "excluded_pairs": 55
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "missed-success-percent",
          "difference_left_minus_right": {
            "value": 0.0,
            "ci95": [
              -6.666666666666666,
              6.666666666666666
            ],
            "observations": 47,
            "independent_groups": 5
          },
          "excluded_pairs": 49
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "generation-seconds",
          "difference_left_minus_right": {
            "value": -1.8048013993982404,
            "ci95": [
              -1.892516824017678,
              -1.7143515394335347
            ],
            "observations": 96,
            "independent_groups": 7
          },
          "excluded_pairs": 0
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "outcome-agreement-percent",
          "difference_left_minus_right": {
            "value": 20.346320346320347,
            "ci95": [
              -32.142857142857146,
              65.58441558441558
            ],
            "observations": 95,
            "independent_groups": 7
          },
          "excluded_pairs": 1
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "visible-outcome-percent",
          "difference_left_minus_right": {
            "value": -1.1904761904761905,
            "ci95": [
              -3.5714285714285716,
              0.0
            ],
            "observations": 96,
            "independent_groups": 7
          },
          "excluded_pairs": 0
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "false-success-percent",
          "difference_left_minus_right": {
            "value": 35.714285714285715,
            "ci95": [
              9.523809523809524,
              60.714285714285715
            ],
            "observations": 45,
            "independent_groups": 7
          },
          "excluded_pairs": 51
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "missed-success-percent",
          "difference_left_minus_right": {
            "value": -97.77777777777777,
            "ci95": [
              -100.0,
              -93.33333333333333
            ],
            "observations": 50,
            "independent_groups": 5
          },
          "excluded_pairs": 46
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "generation-seconds",
          "difference_left_minus_right": {
            "value": null,
            "ci95": null,
            "observations": 0,
            "independent_groups": 0
          },
          "excluded_pairs": 96
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "outcome-agreement-percent",
          "difference_left_minus_right": {
            "value": 25.173160173160174,
            "ci95": [
              -28.051948051948052,
              69.26406926406926
            ],
            "observations": 89,
            "independent_groups": 7
          },
          "excluded_pairs": 7
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "visible-outcome-percent",
          "difference_left_minus_right": {
            "value": -7.142857142857143,
            "ci95": [
              -10.714285714285715,
              -3.5714285714285716
            ],
            "observations": 96,
            "independent_groups": 7
          },
          "excluded_pairs": 0
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "false-success-percent",
          "difference_left_minus_right": {
            "value": 29.285714285714285,
            "ci95": [
              8.571428571428571,
              57.142857142857146
            ],
            "observations": 41,
            "independent_groups": 7
          },
          "excluded_pairs": 55
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "missed-success-percent",
          "difference_left_minus_right": {
            "value": -97.77777777777777,
            "ci95": [
              -100.0,
              -93.33333333333333
            ],
            "observations": 48,
            "independent_groups": 5
          },
          "excluded_pairs": 48
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "generation-seconds",
          "difference_left_minus_right": {
            "value": null,
            "ci95": null,
            "observations": 0,
            "independent_groups": 0
          },
          "excluded_pairs": 96
        }
      ]
    },
    "position_pair": {
      "group_count": 6,
      "members": {
        "position-pair-0": [
          "scene-202610150-center-close",
          "scene-202610150-left-miss",
          "scene-202610150-right-miss",
          "scene-202610150-center-open"
        ],
        "position-pair-4": [
          "scene-202610151-center-close",
          "scene-202610151-left-miss",
          "scene-202610151-right-miss",
          "scene-202610151-center-open",
          "scene-202610154-center-close",
          "scene-202610154-left-miss",
          "scene-202610154-right-miss",
          "scene-202610154-center-open",
          "scene-202610157-center-close",
          "scene-202610157-left-miss",
          "scene-202610157-right-miss",
          "scene-202610157-center-open"
        ],
        "position-pair-11": [
          "scene-202610152-center-close",
          "scene-202610152-left-miss",
          "scene-202610152-right-miss",
          "scene-202610152-center-open"
        ],
        "position-pair-5": [
          "scene-202610153-center-close",
          "scene-202610153-left-miss",
          "scene-202610153-right-miss",
          "scene-202610153-center-open"
        ],
        "position-pair-8": [
          "scene-202610155-center-close",
          "scene-202610155-left-miss",
          "scene-202610155-right-miss",
          "scene-202610155-center-open"
        ],
        "position-pair-6": [
          "scene-202610156-center-close",
          "scene-202610156-left-miss",
          "scene-202610156-right-miss",
          "scene-202610156-center-open"
        ]
      },
      "summaries": [
        {
          "model_id": "cosmos3-edge-robot-action",
          "attempted": 96,
          "succeeded": 96,
          "failed": 0,
          "metrics": {
            "outcome-agreement-percent": {
              "value": 74.4949494949495,
              "ci95": [
                52.65151515151515,
                93.05555555555556
              ],
              "observations": 95,
              "independent_groups": 6
            },
            "visible-outcome-percent": {
              "value": 98.61111111111111,
              "ci95": [
                95.83333333333334,
                100.0
              ],
              "observations": 96,
              "independent_groups": 6
            },
            "false-success-percent": {
              "value": 37.5,
              "ci95": [
                12.5,
                61.111111111111114
              ],
              "observations": 45,
              "independent_groups": 6
            },
            "missed-success-percent": {
              "value": 2.2222222222222223,
              "ci95": [
                0.0,
                6.666666666666666
              ],
              "observations": 50,
              "independent_groups": 5
            },
            "generation-seconds": {
              "value": 8.176708395933492,
              "ci95": [
                8.102654659784296,
                8.258222126906428
              ],
              "observations": 96,
              "independent_groups": 6
            }
          },
          "total_duration_seconds": 885.3157466771081
        },
        {
          "model_id": "cosmos3-nano-robot-action",
          "attempted": 96,
          "succeeded": 96,
          "failed": 0,
          "metrics": {
            "outcome-agreement-percent": {
              "value": 77.91982323232324,
              "ci95": [
                55.17676767676768,
                94.96527777777777
              ],
              "observations": 89,
              "independent_groups": 6
            },
            "visible-outcome-percent": {
              "value": 93.98148148148148,
              "ci95": [
                90.74074074074075,
                97.22222222222223
              ],
              "observations": 96,
              "independent_groups": 6
            },
            "false-success-percent": {
              "value": 29.791666666666668,
              "ci95": [
                7.291666666666667,
                54.791666666666664
              ],
              "observations": 41,
              "independent_groups": 6
            },
            "missed-success-percent": {
              "value": 2.2222222222222223,
              "ci95": [
                0.0,
                6.666666666666666
              ],
              "observations": 48,
              "independent_groups": 5
            },
            "generation-seconds": {
              "value": 9.988901306603415,
              "ci95": [
                9.938928115569677,
                10.0471058686833
              ],
              "observations": 96,
              "independent_groups": 6
            }
          },
          "total_duration_seconds": 1055.8536805948243
        },
        {
          "model_id": "frozen-image",
          "attempted": 96,
          "succeeded": 96,
          "failed": 0,
          "metrics": {
            "outcome-agreement-percent": {
              "value": 45.833333333333336,
              "ci95": [
                29.166666666666668,
                70.83333333333333
              ],
              "observations": 96,
              "independent_groups": 6
            },
            "visible-outcome-percent": {
              "value": 100.0,
              "ci95": [
                100.0,
                100.0
              ],
              "observations": 96,
              "independent_groups": 6
            },
            "false-success-percent": {
              "value": 0.0,
              "ci95": [
                0.0,
                0.0
              ],
              "observations": 45,
              "independent_groups": 6
            },
            "missed-success-percent": {
              "value": 100.0,
              "ci95": [
                100.0,
                100.0
              ],
              "observations": 51,
              "independent_groups": 5
            },
            "generation-seconds": {
              "value": null,
              "ci95": null,
              "observations": 0,
              "independent_groups": 0
            }
          },
          "total_duration_seconds": 0.0
        }
      ],
      "comparisons": [
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "outcome-agreement-percent",
          "difference_left_minus_right": {
            "value": -0.1262626262626263,
            "ci95": [
              -4.545454545454546,
              4.166666666666667
            ],
            "observations": 88,
            "independent_groups": 6
          },
          "excluded_pairs": 8
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "visible-outcome-percent",
          "difference_left_minus_right": {
            "value": 4.62962962962963,
            "ci95": [
              1.388888888888889,
              8.333333333333334
            ],
            "observations": 96,
            "independent_groups": 6
          },
          "excluded_pairs": 0
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "false-success-percent",
          "difference_left_minus_right": {
            "value": 0.0,
            "ci95": [
              0.0,
              0.0
            ],
            "observations": 41,
            "independent_groups": 6
          },
          "excluded_pairs": 55
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "missed-success-percent",
          "difference_left_minus_right": {
            "value": 0.0,
            "ci95": [
              -6.666666666666666,
              6.666666666666666
            ],
            "observations": 47,
            "independent_groups": 5
          },
          "excluded_pairs": 49
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "cosmos3-nano-robot-action",
          "metric": "generation-seconds",
          "difference_left_minus_right": {
            "value": -1.8121929106699235,
            "ci95": [
              -1.9083698689506423,
              -1.706316199179532
            ],
            "observations": 96,
            "independent_groups": 6
          },
          "excluded_pairs": 0
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "outcome-agreement-percent",
          "difference_left_minus_right": {
            "value": 27.904040404040405,
            "ci95": [
              -16.666666666666668,
              64.01515151515152
            ],
            "observations": 95,
            "independent_groups": 6
          },
          "excluded_pairs": 1
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "visible-outcome-percent",
          "difference_left_minus_right": {
            "value": -1.388888888888889,
            "ci95": [
              -4.166666666666667,
              0.0
            ],
            "observations": 96,
            "independent_groups": 6
          },
          "excluded_pairs": 0
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "false-success-percent",
          "difference_left_minus_right": {
            "value": 37.5,
            "ci95": [
              12.5,
              61.111111111111114
            ],
            "observations": 45,
            "independent_groups": 6
          },
          "excluded_pairs": 51
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "missed-success-percent",
          "difference_left_minus_right": {
            "value": -97.77777777777777,
            "ci95": [
              -100.0,
              -93.33333333333333
            ],
            "observations": 50,
            "independent_groups": 5
          },
          "excluded_pairs": 46
        },
        {
          "left": "cosmos3-edge-robot-action",
          "right": "frozen-image",
          "metric": "generation-seconds",
          "difference_left_minus_right": {
            "value": null,
            "ci95": null,
            "observations": 0,
            "independent_groups": 0
          },
          "excluded_pairs": 96
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "outcome-agreement-percent",
          "difference_left_minus_right": {
            "value": 33.60164141414141,
            "ci95": [
              -14.488636363636365,
              67.81881313131314
            ],
            "observations": 89,
            "independent_groups": 6
          },
          "excluded_pairs": 7
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "visible-outcome-percent",
          "difference_left_minus_right": {
            "value": -6.018518518518519,
            "ci95": [
              -9.25925925925926,
              -2.777777777777778
            ],
            "observations": 96,
            "independent_groups": 6
          },
          "excluded_pairs": 0
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "false-success-percent",
          "difference_left_minus_right": {
            "value": 29.791666666666668,
            "ci95": [
              7.291666666666667,
              54.791666666666664
            ],
            "observations": 41,
            "independent_groups": 6
          },
          "excluded_pairs": 55
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "missed-success-percent",
          "difference_left_minus_right": {
            "value": -97.77777777777777,
            "ci95": [
              -100.0,
              -93.33333333333333
            ],
            "observations": 48,
            "independent_groups": 5
          },
          "excluded_pairs": 48
        },
        {
          "left": "cosmos3-nano-robot-action",
          "right": "frozen-image",
          "metric": "generation-seconds",
          "difference_left_minus_right": {
            "value": null,
            "ci95": null,
            "observations": 0,
            "independent_groups": 0
          },
          "excluded_pairs": 96
        }
      ]
    }
  },
  "layout_audit": {
    "url": "https://rather-media-143952026851-us-west-2.s3.us-west-2.amazonaws.com/public/studies/assets/799606e75d9017d73e991b14f72749931b82d71db45b7ce544fdb38b7c8f4011/robot-layout-audit-v2.json",
    "sha256": "799606e75d9017d73e991b14f72749931b82d71db45b7ce544fdb38b7c8f4011"
  },
  "analysis_code": {
    "url": "https://rather-media-143952026851-us-west-2.s3.us-west-2.amazonaws.com/public/studies/assets/153c7f184816a04d83447624d23cb40eba3af161e7522a08daee0816209573a0/analyze_robot_layout_correction_v2.py",
    "sha256": "153c7f184816a04d83447624d23cb40eba3af161e7522a08daee0816209573a0"
  }
}