{
  "id": "rather-camera-return-v1",
  "status": "registered",
  "prepared_at": "2026-10-16T02:14:07.730808+00:00",
  "kind": "rather_diagnostic",
  "question": "Which configuration responds to camera controls, returns to a coherent view, and stays steady after input stops?",
  "dataset": {
    "repository": "meituan-longcat/WBench",
    "revision": "bb6dee5a674cc5cb3019f9c79d4871899fd9e04c",
    "license": "MIT",
    "license_sha256": "f283a6624e6bf788beb5f18b925fc97c6c646de49dc451cb7a857b1ca2f16254",
    "attribution": "WBench authors / Meituan LongCat and Fudan University",
    "selection": "First two numeric-ID navigation-only first-person cases in each of Indoor, Nature, Urban and Fantasy; 8 scenes selected before generation from 289 source records."
  },
  "seeds": [
    202610160,
    202610161,
    202610162
  ],
  "frames": 177,
  "playback_fps": 16,
  "conditioning": {
    "image": "Shared RGB image, center cropped to 20:11 and resized to 1280x704. Matrix-Game 2 uses its native 640x352 downsample.",
    "text": "Empty for every configuration; no off-screen descriptions or control instructions in text."
  },
  "models": [
    "matrix-game3-base",
    "matrix-game3-distilled",
    "matrix-game2-universal",
    "frozen-frame-baseline"
  ],
  "metrics": [
    "action-following-percent",
    "stationary-drift-pixels",
    "return-view-ssim",
    "frozen-action-percent",
    "motion-coverage-percent",
    "generation-seconds",
    "generation-frames-per-second",
    "generation-gpu-usd"
  ],
  "limitations": [
    "Eight selected scenes do not establish broad generalization.",
    "Image-plane motion and return-view SSIM are diagnostic proxies, not physical state or semantic-memory ground truth.",
    "Dynamic objects, low texture and scene changes can make motion estimation undefined; coverage and missing values must be reported.",
    "Equal and opposite native mouse input requests a return, but does not guarantee the generated camera actually returns.",
    "The frozen baseline intentionally scores highly on return similarity while failing action response.",
    "This is a new Rather protocol using licensed WBench images, not a WBench leaderboard score.",
    "Offline generation timing is not end-to-end interactive service latency.",
    "Three configurations of one Skywork model family, not broad provider coverage.",
    "Released native resolutions differ. Image-plane metrics use the same 640x352 canvas; compute results describe each native configuration, not equal-resolution ablations.",
    "Matrix-Game 3 uses explicitly recorded PyTorch 2.8 / Flash Attention 2 instead of the upstream Torch 2.10 pin, with unchanged pinned model code. This runtime passed native generation probes.",
    "The museum left-return scene was used in one Matrix-Game 2 connector probe with a different seed. All images were used for metric calibration; no held-out-development claim.",
    "The 11.06-second horizon checks a short camera-return interaction, not long-horizon memory. No semantic, 3D-state, real-time-serving or physical-consistency claim is made.",
    "Frozen-image output is reused across seeds and does not add independent baseline repetitions."
  ],
  "cases": [
    {
      "id": "wbench-5-left-return",
      "group": "wbench-5",
      "label": "museum gallery / left return",
      "path": "scene-5/left-return.json",
      "sha256": "7e1bc40b597873f0ab24f702889b7071d1894ab7442e9600259cc6d2f7e25788"
    },
    {
      "id": "wbench-5-right-return",
      "group": "wbench-5",
      "label": "museum gallery / right return",
      "path": "scene-5/right-return.json",
      "sha256": "7a75eac155dc942d8e4252e773f48e0e678e4b1b68c2ec3f47c4cfd99cea5c5d"
    },
    {
      "id": "wbench-12-left-return",
      "group": "wbench-12",
      "label": "supermarket / left return",
      "path": "scene-12/left-return.json",
      "sha256": "8a23dd0632e86134584d5aa2a20c197639ff178b8a05e75df96ec0467559f076"
    },
    {
      "id": "wbench-12-right-return",
      "group": "wbench-12",
      "label": "supermarket / right return",
      "path": "scene-12/right-return.json",
      "sha256": "db491218f432a1db022fbee4e3226afaa52d16f2a53fea7acc2ffe6c39d20e9e"
    },
    {
      "id": "wbench-34-left-return",
      "group": "wbench-34",
      "label": "desert campfire / left return",
      "path": "scene-34/left-return.json",
      "sha256": "2441f3f61b3a56ccece3d7200349040960815c44351b85ff0574654965da6d81"
    },
    {
      "id": "wbench-34-right-return",
      "group": "wbench-34",
      "label": "desert campfire / right return",
      "path": "scene-34/right-return.json",
      "sha256": "673e1eb26f58f7c2ef3139fcd2473714e642e264f9fd3eb4bbbb62232b99ba31"
    },
    {
      "id": "wbench-35-left-return",
      "group": "wbench-35",
      "label": "bamboo forest / left return",
      "path": "scene-35/left-return.json",
      "sha256": "a7178969776b9f7b970c8137f277f2a0639bf5a515b3cfec9e400ac9c99a0fc7"
    },
    {
      "id": "wbench-35-right-return",
      "group": "wbench-35",
      "label": "bamboo forest / right return",
      "path": "scene-35/right-return.json",
      "sha256": "f131049850e6f1a58f196f50d1f4ea79478bdd063751d6abf1bb08521c440ffe"
    },
    {
      "id": "wbench-36-left-return",
      "group": "wbench-36",
      "label": "skyscraper rooftop / left return",
      "path": "scene-36/left-return.json",
      "sha256": "9b19f850aaac7c2ec57b2ac5e6c9558f692c4eb9afa23d86637eed8ca0b74960"
    },
    {
      "id": "wbench-36-right-return",
      "group": "wbench-36",
      "label": "skyscraper rooftop / right return",
      "path": "scene-36/right-return.json",
      "sha256": "954aa887f51ce42851b544cc34789ca9d7fbae7cd00d3eb07c668215af12bb7d"
    },
    {
      "id": "wbench-40-left-return",
      "group": "wbench-40",
      "label": "japanese temple / left return",
      "path": "scene-40/left-return.json",
      "sha256": "eea2da938de981aa3eb120a16328546fe4108338f580282fe55f67ef5935c654"
    },
    {
      "id": "wbench-40-right-return",
      "group": "wbench-40",
      "label": "japanese temple / right return",
      "path": "scene-40/right-return.json",
      "sha256": "4364d4b8df23b22bf5a106d179719680453bfbc29915552608374ed74f934546"
    },
    {
      "id": "wbench-79-left-return",
      "group": "wbench-79",
      "label": "misty lake / left return",
      "path": "scene-79/left-return.json",
      "sha256": "8594e2299fcf09aba003c7361ae13f53b8bb05d27c2a6ee329a37c492627e29c"
    },
    {
      "id": "wbench-79-right-return",
      "group": "wbench-79",
      "label": "misty lake / right return",
      "path": "scene-79/right-return.json",
      "sha256": "debe6300a17013ce283dd5b4b2e244681e84fd5134edef4204591a9d427eca65"
    },
    {
      "id": "wbench-80-left-return",
      "group": "wbench-80",
      "label": "spaceship corridor / left return",
      "path": "scene-80/left-return.json",
      "sha256": "e4c58df50209057e345aff8b7adda4b965f869a41e4550adaf4595fbfebd3b08"
    },
    {
      "id": "wbench-80-right-return",
      "group": "wbench-80",
      "label": "spaceship corridor / right return",
      "path": "scene-80/right-return.json",
      "sha256": "95dd6070be0319b21da549492ef5efcd139d7a315115350ab66503cb343be739"
    }
  ],
  "registered_at": "2026-10-16T03:52:01.016889+00:00",
  "hardware": "One NVIDIA H100 80GB HBM3 (SXM) per worker, same hardware class for all three model configurations; CPU scoring is separate.",
  "execution_shards": {
    "matrix-game3-base": [
      [
        0,
        4
      ],
      [
        4,
        8
      ],
      [
        8,
        12
      ],
      [
        12,
        16
      ]
    ],
    "matrix-game3-distilled": [
      [
        0,
        16
      ]
    ],
    "matrix-game2-universal": [
      [
        0,
        16
      ]
    ]
  },
  "runner_scripts": {
    "matrix-game3-base": {
      "scripts/run_game3_study.py": "d43b1e8ea09b63817b4e8b1446178c832468e478e9f2a81b6901b9e85feb68c5",
      "scripts/run_game3_shard.sh": "2c912bcaf7701b2edf8aaafc0ccb3b46ab05b3127716978d227db593a49cc479",
      "scripts/eval_game3_torch28.sh": "e02e74694312b514820adf5c519c81ea702002ab5e3bd9a25e907c28cd2e8a2b",
      "data/evaluation/game3-torch28-environment.txt": "b1c77a38c0b8624a48b0d122beebf7d33b2c864b44c942317a20b950131dadee"
    },
    "matrix-game3-distilled": {
      "scripts/run_game3_study.py": "d43b1e8ea09b63817b4e8b1446178c832468e478e9f2a81b6901b9e85feb68c5",
      "scripts/run_game3_shard.sh": "2c912bcaf7701b2edf8aaafc0ccb3b46ab05b3127716978d227db593a49cc479",
      "scripts/eval_game3_torch28.sh": "e02e74694312b514820adf5c519c81ea702002ab5e3bd9a25e907c28cd2e8a2b",
      "data/evaluation/game3-torch28-environment.txt": "b1c77a38c0b8624a48b0d122beebf7d33b2c864b44c942317a20b950131dadee"
    },
    "matrix-game2-universal": {
      "scripts/run_game2_study.py": "a2db60c579f26b602137c2f88f8fed760bece5dd16bb4e99db84c2c8dc4b2a6b",
      "scripts/run_game2_study.sh": "3792faa2189da8865be403c9780dfbbd69687ae2ae7c4a2f1fb4b73932a3b2fa",
      "scripts/eval_game2_probe.sh": "fdd0bae49a9d6054f05343fac483657e236a745899e9f0c7c9d6b25b976ff2f5",
      "scripts/eval_game_probe.py": "c8866603a7ff34a7a71b7ef6207367b609c4ae17891b769b59af1d3b4707448b",
      "data/evaluation/game2-torch28-environment.txt": "48a51e176abd1caeed10e6adaa32cb72551dccf41af15ff06c2ff049f85d3915"
    }
  },
  "evaluator": {
    "script_sha256": "d3129de432acf49f31c1e260e6c9649f8366036f14b35674416a6ed74a2a0290",
    "runtime": {
      "numpy": "2.2.6",
      "opencv-python-headless": "4.10.0.84",
      "scikit-image": "0.26.0",
      "imageio": "2.38.1",
      "Pillow": "12.3.0"
    },
    "motion_calibration_sha256": "19b6a2ec28d4a86814cd4a13e79b7272ad69ff172172fe24fcdd2cda5f299ee6",
    "baseline_calibration_sha256": "f747f64bd29adec3befea74a0460164fecc537b40f2d08f08d005af53ea8800b"
  },
  "native_probe_evidence": {
    "artifacts/games-native-probe-torch28-20261016/results/base/attempt.json": "97497cdfb4f99f3e207def6b08c37130d2bb0e2e1879452f9d5df4e94cb1864e",
    "artifacts/games-native-probe-torch28-20261016/results/distilled/attempt.json": "d2b231d248840cfaf293f235595e7cbd8de410a7163bc846cb7f9ff215f52fc3",
    "artifacts/games-matrix2-native-probe-20261015/results/universal/attempt.json": "3311183ebd60f9b12ed34d31dbe84430eb484b7d8b3661fdb08c9ead144bd590"
  },
  "uncertainty": "Equal scene weights over eight scene groups. 2,000 scene-cluster bootstrap draws, seed 20261015. Three seeds and two turn directions stay within each scene cluster. Paired differences use jointly scored attempts only.",
  "failure_policy": "Exactly one attempt per configuration, case and seed. Preserve errors and unstarted/deadline failures, no hidden retries. Undefined motion estimates remain missing with coverage; never zero-fill.",
  "example_selection": "First registered seed for every case, in preselected input order. No cherry-picking by metric or appearance.",
  "timing": "Wall time around each synchronized native pipeline invocation includes preprocessing and native video writing; excludes installation and checkpoint loading. Effective frames/s is 177 divided by that duration; warm GPU cost uses the actual recorded hourly price.",
  "preflight_jobs": [
    "games-matrix2-native-probe-20261015",
    "games-native-probe-20261015",
    "games-native-probe-flash2-20261015",
    "games-native-probe-flash2-bounded-20261015",
    "games-native-probe-retry-20261015",
    "games-native-probe-torch28-20261016"
  ],
  "configurations": {
    "matrix-game3-base": {
      "source_revision": "ad397b34dd78880bad068232227a49f7106af817",
      "checkpoint_revision": "382aa382a03cec057360755aa72c7735025770bb",
      "steps": 50,
      "torch": "2.8.0+cu128",
      "flash_attention": "2.8.3",
      "native_resolution": [
        1280,
        704
      ],
      "vae": "synchronous MG-LightVAE v2, pruning 0.75, no compile/async/int8",
      "guidance": 5.0,
      "sample_shift": 5.0
    },
    "matrix-game3-distilled": {
      "source_revision": "ad397b34dd78880bad068232227a49f7106af817",
      "checkpoint_revision": "382aa382a03cec057360755aa72c7735025770bb",
      "steps": 3,
      "torch": "2.8.0+cu128",
      "flash_attention": "2.8.3",
      "native_resolution": [
        1280,
        704
      ],
      "vae": "synchronous MG-LightVAE v2, pruning 0.75, no compile/async/int8",
      "guidance": 5.0,
      "sample_shift": 5.0
    },
    "matrix-game2-universal": {
      "source_revision": "ad397b34dd78880bad068232227a49f7106af817",
      "checkpoint_revision": "08c42d1a99e57e79df60a964bb815618cdf460f8",
      "steps": 3,
      "torch": "2.8.0+cu128",
      "flash_attention": "2.8.3",
      "native_resolution": [
        640,
        352
      ],
      "vae": "native eager FP16 VAE, BF16 denoiser",
      "guidance": "native distilled sampler",
      "sample_shift": "native sampler"
    }
  },
  "execution_jobs": {
    "games-return-v1-mg3-base-0-20261016": {
      "model_id": "matrix-game3-base",
      "case_partition": [
        0,
        4
      ]
    },
    "games-return-v1-mg3-base-1-20261016": {
      "model_id": "matrix-game3-base",
      "case_partition": [
        4,
        8
      ]
    },
    "games-return-v1-mg3-base-2-20261016": {
      "model_id": "matrix-game3-base",
      "case_partition": [
        8,
        12
      ]
    },
    "games-return-v1-mg3-base-3-20261016": {
      "model_id": "matrix-game3-base",
      "case_partition": [
        12,
        16
      ]
    },
    "games-return-v1-mg3-distilled-0-20261016": {
      "model_id": "matrix-game3-distilled",
      "case_partition": [
        0,
        16
      ]
    },
    "games-return-v1-mg2-universal-0-20261016": {
      "model_id": "matrix-game2-universal",
      "case_partition": [
        0,
        16
      ]
    }
  },
  "registration_revision_note": {
    "supersedes_sha256": "43fe59ba7678b50510a28d1fed8648cc574a051d3e2b4a3e359b0c94179dc2f4",
    "reason": "Canonical LF line endings for cross-platform checksum reproducibility; corrected before creating any scored games rental. No inputs, metrics, model settings, seeds or partition changed."
  }
}