{
  "status": "posthoc sensitivity diagnostic; not a new model experiment or replacement official score",
  "method": "For every arm, locate a unique schema-valid entities/relations payload inside the recorded submission, including JSON strings; strip surrounding wrappers/extra metadata only. Preserve all entity/relation content and source grading. Missing or ambiguous output retains original score. No gold is used to select a payload.",
  "limitation": "The rule was chosen after inspecting failures. This checks sensitivity to format handling; it does not measure prospective model recovery or establish a skill winner.",
  "sources_sha256": {
    "runs/four-track-canary-v3/manifest.json": "97869f65237bb757bf688e9819a45743bc78c7b98355d416393f2326df824daf",
    "runs/four-track-canary-v3/results.json": "c54bac920410a03218abf2a0a5a8aa99472a740ef205f36a011f8094c6673591",
    "runs/new-sources-and-analysis-v3/manifest.json": "0251dbb5a3dc40199ddfd4960403b4f58c65d95029eeb2bd74d9f7237bdd172e",
    "runs/new-sources-and-analysis-v3/results.json": "987d34b2c46f2ba591e07c2f88687099a9077465ea1e839bc4910869a8cb285c"
  },
  "grader_sha256": "9601e4cb55b694ba79442eed51275431676acbe5124fd8aa6d23d67481fb3d00",
  "arms": {
    "baseline": {
      "n": 8,
      "original_score": 0.23965531275876104,
      "diagnostic_score": 0.3997261154367797,
      "rescored_outputs": 3
    },
    "checklist": {
      "n": 8,
      "original_score": 0.30550972247342745,
      "diagnostic_score": 0.34145294591665093,
      "rescored_outputs": 1
    },
    "skill": {
      "n": 8,
      "original_score": 0.3343047793039286,
      "diagnostic_score": 0.39085239835154767,
      "rescored_outputs": 1
    }
  },
  "rows": [
    {
      "task": "extract-scierc-ICCV_2003_151_abs",
      "arm": "baseline",
      "status": "completed",
      "original_score": 0.0,
      "original_schema_valid": 0.0,
      "diagnostic_score": 0.27363737486095663,
      "unique_valid_payload": true,
      "changed": true
    },
    {
      "task": "extract-scierc-ICCV_2003_151_abs",
      "arm": "checklist",
      "status": "completed",
      "original_score": 0.34937124111536355,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.34937124111536355,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-ICCV_2003_151_abs",
      "arm": "skill",
      "status": "completed",
      "original_score": 0.28673245614035087,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.28673245614035087,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-C94-1091",
      "arm": "baseline",
      "status": "completed",
      "original_score": 0.5,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.5,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-C94-1091",
      "arm": "checklist",
      "status": "completed",
      "original_score": 0.5,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.5,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-C94-1091",
      "arm": "skill",
      "status": "completed",
      "original_score": 0.3992740471869328,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.3992740471869328,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-P01-1056",
      "arm": "baseline",
      "status": "completed",
      "original_score": 0.37362637362637363,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.37362637362637363,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-P01-1056",
      "arm": "checklist",
      "status": "completed",
      "original_score": 0.0,
      "original_schema_valid": 0.0,
      "diagnostic_score": 0.2875457875457875,
      "unique_valid_payload": true,
      "changed": true
    },
    {
      "task": "extract-scierc-P01-1056",
      "arm": "skill",
      "status": "completed",
      "original_score": 0.46865203761755486,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.46865203761755486,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-ICCV_2001_47_abs",
      "arm": "baseline",
      "status": "completed",
      "original_score": 0.0,
      "original_schema_valid": 0.0,
      "diagnostic_score": 0.30995934959349597,
      "unique_valid_payload": true,
      "changed": true
    },
    {
      "task": "extract-scierc-ICCV_2001_47_abs",
      "arm": "checklist",
      "status": "completed",
      "original_score": 0.2916666666666667,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.2916666666666667,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-ICCV_2001_47_abs",
      "arm": "skill",
      "status": "completed",
      "original_score": 0.0,
      "original_schema_valid": 0.0,
      "diagnostic_score": 0.4523809523809524,
      "unique_valid_payload": true,
      "changed": true
    },
    {
      "task": "extract-scierc-CVPR_1994_10_abs",
      "arm": "baseline",
      "status": "completed",
      "original_score": 0.4735449735449735,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.4735449735449735,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-CVPR_1994_10_abs",
      "arm": "checklist",
      "status": "completed",
      "original_score": 0.5490716180371353,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.5490716180371353,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-CVPR_1994_10_abs",
      "arm": "skill",
      "status": "completed",
      "original_score": 0.4735632183908046,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.4735632183908046,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-X96-1059",
      "arm": "baseline",
      "status": "completed",
      "original_score": 0.3155555555555556,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.3155555555555556,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-X96-1059",
      "arm": "checklist",
      "status": "completed",
      "original_score": 0.4095238095238095,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.4095238095238095,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-X96-1059",
      "arm": "skill",
      "status": "completed",
      "original_score": 0.3670977011494253,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.3670977011494253,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-E91-1012",
      "arm": "baseline",
      "status": "completed",
      "original_score": 0.2545155993431856,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.2545155993431856,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-E91-1012",
      "arm": "checklist",
      "status": "completed",
      "original_score": 0.34444444444444444,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.34444444444444444,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-E91-1012",
      "arm": "skill",
      "status": "completed",
      "original_score": 0.2624521072796935,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.2624521072796935,
      "unique_valid_payload": true,
      "changed": false
    },
    {
      "task": "extract-scierc-P06-1013",
      "arm": "baseline",
      "status": "completed",
      "original_score": 0.0,
      "original_schema_valid": 0.0,
      "diagnostic_score": 0.696969696969697,
      "unique_valid_payload": true,
      "changed": true
    },
    {
      "task": "extract-scierc-P06-1013",
      "arm": "checklist",
      "status": "failed",
      "original_score": 0.0,
      "original_schema_valid": null,
      "diagnostic_score": 0.0,
      "unique_valid_payload": false,
      "changed": false
    },
    {
      "task": "extract-scierc-P06-1013",
      "arm": "skill",
      "status": "completed",
      "original_score": 0.41666666666666663,
      "original_schema_valid": 1.0,
      "diagnostic_score": 0.41666666666666663,
      "unique_valid_payload": true,
      "changed": false
    }
  ],
  "model_calls": 0,
  "incremental_model_cost_usd": 0
}