{
  "primary_pass": false,
  "pass_checks": {
    "point_estimates": false,
    "accuracy_interval": true,
    "timing_interval": true,
    "at_least_two_seeds": true,
    "all_calibrations_feasible": true
  },
  "passing_seed_count": 2,
  "aggregate": {
    "unchanged": {
      "accuracy": {
        "mean": 0.7740885416666666,
        "ci95": [
          0.7573201497395833,
          0.7911783854166667
        ]
      },
      "valid_fraction": 0.99951171875,
      "mean_repeats": 3.0983072916666665,
      "mean_stack_applications": 6.196614583333333,
      "seconds": {
        "mean": 0.14610546499461635,
        "ci95": [
          0.13814536367123464,
          0.15412003878762334
        ]
      },
      "full_test_seconds": 0.15160326916050812,
      "by_task": {
        "boolq": {
          "accuracy": 0.7955729166666666,
          "mean_repeats": 3.2194010416666665,
          "mean_seconds": 0.1523196524028511
        },
        "piqa": {
          "accuracy": 0.7109375,
          "mean_repeats": 3.555989583333333,
          "mean_seconds": 0.15988156263483688
        },
        "arc": {
          "accuracy": 0.8352864583333333,
          "mean_repeats": 2.7434895833333335,
          "mean_seconds": 0.13546533350211878
        },
        "tracking": {
          "accuracy": 0.7545572916666667,
          "mean_repeats": 2.874348958333333,
          "mean_seconds": 0.1367553114386586
        }
      }
    },
    "unchanged-fixed4": {
      "accuracy": {
        "mean": 0.7801106770833333,
        "ci95": [
          0.7638305664062499,
          0.796875
        ]
      },
      "valid_fraction": 0.99951171875,
      "mean_repeats": 4,
      "mean_stack_applications": 8,
      "seconds": {
        "mean": 0.19483544658238922,
        "ci95": [
          0.1947135943652635,
          0.19495515202826938
        ]
      },
      "full_test_seconds": 0.19438763482730792,
      "by_task": {
        "boolq": {
          "accuracy": 0.8001302083333333,
          "mean_repeats": 4,
          "mean_seconds": 0.1955340524339893
        },
        "piqa": {
          "accuracy": 0.7115885416666666,
          "mean_repeats": 4,
          "mean_seconds": 0.1943204087593282
        },
        "arc": {
          "accuracy": 0.8515625,
          "mean_repeats": 4,
          "mean_seconds": 0.19465822414301026
        },
        "tracking": {
          "accuracy": 0.7571614583333333,
          "mean_repeats": 4,
          "mean_seconds": 0.1948291009932291
        }
      }
    },
    "final-only": {
      "accuracy": {
        "mean": 0.8444010416666666,
        "ci95": [
          0.8310546875,
          0.8582356770833333
        ]
      },
      "valid_fraction": 1,
      "mean_repeats": 2.1984049479166665,
      "mean_stack_applications": 4.396809895833333,
      "seconds": {
        "mean": 0.10438154191409314,
        "ci95": [
          0.09785839428161125,
          0.1113151493200227
        ]
      },
      "full_test_seconds": 0.1081716214412154,
      "by_task": {
        "boolq": {
          "accuracy": 0.8567708333333333,
          "mean_repeats": 2.40625,
          "mean_seconds": 0.11694126668735408
        },
        "piqa": {
          "accuracy": 0.80859375,
          "mean_repeats": 2.565755208333333,
          "mean_seconds": 0.11255162771364363
        },
        "arc": {
          "accuracy": 0.81640625,
          "mean_repeats": 1.9973958333333333,
          "mean_seconds": 0.09264434569825727
        },
        "tracking": {
          "accuracy": 0.8958333333333333,
          "mean_repeats": 1.82421875,
          "mean_seconds": 0.09538892755711761
        }
      }
    },
    "uniform": {
      "accuracy": {
        "mean": 0.85009765625,
        "ci95": [
          0.8361002604166667,
          0.8639322916666667
        ]
      },
      "valid_fraction": 1,
      "mean_repeats": 1.6822916666666667,
      "mean_stack_applications": 3.3645833333333335,
      "seconds": {
        "mean": 0.07999321842604938,
        "ci95": [
          0.0748299136271089,
          0.08547539704153072
        ]
      },
      "full_test_seconds": 0.08323190486392681,
      "by_task": {
        "boolq": {
          "accuracy": 0.8307291666666666,
          "mean_repeats": 1.674479166666667,
          "mean_seconds": 0.09159024954036188
        },
        "piqa": {
          "accuracy": 0.8248697916666666,
          "mean_repeats": 2.044921875,
          "mean_seconds": 0.0848849176953081
        },
        "arc": {
          "accuracy": 0.8203124999999999,
          "mean_repeats": 1.7063802083333335,
          "mean_seconds": 0.0810324781263868
        },
        "tracking": {
          "accuracy": 0.9244791666666667,
          "mean_repeats": 1.3033854166666667,
          "mean_seconds": 0.062465228342140716
        }
      }
    },
    "early-weighted": {
      "accuracy": {
        "mean": 0.8517252604166666,
        "ci95": [
          0.8377278645833333,
          0.8658894856770835
        ]
      },
      "valid_fraction": 1,
      "mean_repeats": 1.7888997395833335,
      "mean_stack_applications": 3.577799479166667,
      "seconds": {
        "mean": 0.08279495510602525,
        "ci95": [
          0.07674076613877939,
          0.08904957806590877
        ]
      },
      "full_test_seconds": 0.08834171456131419,
      "by_task": {
        "boolq": {
          "accuracy": 0.8424479166666667,
          "mean_repeats": 1.876953125,
          "mean_seconds": 0.10075037862601069
        },
        "piqa": {
          "accuracy": 0.8157552083333334,
          "mean_repeats": 2.234375,
          "mean_seconds": 0.08842809231136925
        },
        "arc": {
          "accuracy": 0.822265625,
          "mean_repeats": 1.7434895833333335,
          "mean_seconds": 0.08303531443428558
        },
        "tracking": {
          "accuracy": 0.9264322916666666,
          "mean_repeats": 1.30078125,
          "mean_seconds": 0.05896603505243547
        }
      }
    }
  },
  "comparisons": {
    "early-weighted_minus_unchanged": {
      "accuracy_difference": {
        "mean": 0.07763671875,
        "ci95": [
          0.060872395833333336,
          0.09359130859375002
        ]
      },
      "time_ratio": {
        "mean": 0.5666793853951736,
        "ci95": [
          0.5291708463683882,
          0.6051172980277958
        ]
      },
      "full_test_time_ratio": 0.5827164219511881,
      "by_task": {
        "boolq": {
          "accuracy_difference": 0.04687500000000011,
          "time_ratio": 0.6614404447270447
        },
        "piqa": {
          "accuracy_difference": 0.10481770833333337,
          "time_ratio": 0.5530849890011114
        },
        "arc": {
          "accuracy_difference": -0.01302083333333326,
          "time_ratio": 0.6129635699969457
        },
        "tracking": {
          "accuracy_difference": 0.1718749999999999,
          "time_ratio": 0.43117912154281923
        }
      }
    },
    "early-weighted_minus_final-only": {
      "accuracy_difference": {
        "mean": 0.007324218750000003,
        "ci95": [
          -0.0029296874999999965,
          0.017578125
        ]
      },
      "time_ratio": {
        "mean": 0.7931953637374525,
        "ci95": [
          0.7459363422585391,
          0.8417939081813265
        ]
      },
      "full_test_time_ratio": 0.8166810609317017,
      "by_task": {
        "boolq": {
          "accuracy_difference": -0.014322916666666519,
          "time_ratio": 0.8615468386824455
        },
        "piqa": {
          "accuracy_difference": 0.00716145833333337,
          "time_ratio": 0.7856669344343022
        },
        "arc": {
          "accuracy_difference": 0.005859375,
          "time_ratio": 0.8962804346930322
        },
        "tracking": {
          "accuracy_difference": 0.03059895833333337,
          "time_ratio": 0.6181643568340508
        }
      }
    },
    "early-weighted_minus_uniform": {
      "accuracy_difference": {
        "mean": 0.0016276041666666665,
        "ci95": [
          -0.003417968749999999,
          0.006673177083333334
        ]
      },
      "time_ratio": {
        "mean": 1.0350246775302079,
        "ci95": [
          1.0047512170373796,
          1.0684740703234887
        ]
      },
      "full_test_time_ratio": 1.0613924396629062,
      "by_task": {
        "boolq": {
          "accuracy_difference": 0.011718750000000111,
          "time_ratio": 1.100012055121786
        },
        "piqa": {
          "accuracy_difference": -0.00911458333333326,
          "time_ratio": 1.0417409206754404
        },
        "arc": {
          "accuracy_difference": 0.001953125000000111,
          "time_ratio": 1.0247164637464863
        },
        "tracking": {
          "accuracy_difference": 0.001953124999999889,
          "time_ratio": 0.9439817418013888
        }
      }
    },
    "uniform_minus_unchanged": {
      "accuracy_difference": {
        "mean": 0.07600911458333334,
        "ci95": [
          0.05956624348958333,
          0.09228515625
        ]
      },
      "time_ratio": {
        "mean": 0.5475032602579031,
        "ci95": [
          0.5161423319363991,
          0.5802583222505141
        ]
      },
      "full_test_time_ratio": 0.5490112800655113,
      "by_task": {
        "boolq": {
          "accuracy_difference": 0.03515625,
          "time_ratio": 0.6013029054066269
        },
        "piqa": {
          "accuracy_difference": 0.11393229166666663,
          "time_ratio": 0.5309237431534358
        },
        "arc": {
          "accuracy_difference": -0.01497395833333337,
          "time_ratio": 0.5981787076552643
        },
        "tracking": {
          "accuracy_difference": 0.169921875,
          "time_ratio": 0.4567663784683011
        }
      }
    },
    "final-only_minus_unchanged": {
      "accuracy_difference": {
        "mean": 0.0703125,
        "ci95": [
          0.05517578125,
          0.08561197916666666
        ]
      },
      "time_ratio": {
        "mean": 0.7144259930177106,
        "ci95": [
          0.6814376387599675,
          0.7495071519342167
        ]
      },
      "full_test_time_ratio": 0.713517736393204,
      "by_task": {
        "boolq": {
          "accuracy_difference": 0.06119791666666663,
          "time_ratio": 0.7677359082862848
        },
        "piqa": {
          "accuracy_difference": 0.09765625,
          "time_ratio": 0.7039687745028304
        },
        "arc": {
          "accuracy_difference": -0.01888020833333326,
          "time_ratio": 0.6838970776003607
        },
        "tracking": {
          "accuracy_difference": 0.14127604166666652,
          "time_ratio": 0.69751534001591
        }
      }
    },
    "unchanged_minus_unchanged-fixed4": {
      "accuracy_difference": {
        "mean": -0.006022135416666668,
        "ci95": [
          -0.009765625,
          -0.0027669270833333335
        ]
      },
      "time_ratio": {
        "mean": 0.7498916011303588,
        "ci95": [
          0.7091049108687884,
          0.7910480009410593
        ]
      },
      "full_test_time_ratio": 0.7799018147177468,
      "by_task": {
        "boolq": {
          "accuracy_difference": -0.00455729166666663,
          "time_ratio": 0.7789929708242148
        },
        "piqa": {
          "accuracy_difference": -0.0006510416666666297,
          "time_ratio": 0.8227728814262382
        },
        "arc": {
          "accuracy_difference": -0.01627604166666674,
          "time_ratio": 0.6959137436833698
        },
        "tracking": {
          "accuracy_difference": -0.0026041666666665186,
          "time_ratio": 0.7019244596494405
        }
      }
    }
  },
  "seeds": {
    "20260904": {
      "unchanged": {
        "accuracy": 0.771484375,
        "mean_repeats": 3.16455078125,
        "mean_seconds": 0.15003482225438347,
        "by_task": {
          "boolq": {
            "accuracy": 0.765625,
            "mean_repeats": 3.208984375,
            "mean_seconds": 0.15169022467453033
          },
          "piqa": {
            "accuracy": 0.71875,
            "mean_repeats": 3.59375,
            "mean_seconds": 0.15822903782245704
          },
          "arc": {
            "accuracy": 0.84765625,
            "mean_repeats": 2.78125,
            "mean_seconds": 0.1404225594480522
          },
          "tracking": {
            "accuracy": 0.75390625,
            "mean_repeats": 3.07421875,
            "mean_seconds": 0.1497974670724943
          }
        }
      },
      "unchanged-fixed4": {
        "accuracy": 0.77587890625,
        "mean_repeats": 4,
        "mean_seconds": 0.19394981968216596,
        "by_task": {
          "boolq": {
            "accuracy": 0.76953125,
            "mean_repeats": 4,
            "mean_seconds": 0.19472470100736247
          },
          "piqa": {
            "accuracy": 0.71875,
            "mean_repeats": 4,
            "mean_seconds": 0.19339359059813432
          },
          "arc": {
            "accuracy": 0.859375,
            "mean_repeats": 4,
            "mean_seconds": 0.19367699325084686
          },
          "tracking": {
            "accuracy": 0.755859375,
            "mean_repeats": 4,
            "mean_seconds": 0.19400399387232029
          }
        }
      },
      "final-only": {
        "accuracy": 0.84130859375,
        "mean_repeats": 1.99072265625,
        "mean_seconds": 0.08899997488188092,
        "by_task": {
          "boolq": {
            "accuracy": 0.8359375,
            "mean_repeats": 2.125,
            "mean_seconds": 0.09576634370605461
          },
          "piqa": {
            "accuracy": 0.818359375,
            "mean_repeats": 2.27734375,
            "mean_seconds": 0.09510387943009846
          },
          "arc": {
            "accuracy": 0.818359375,
            "mean_repeats": 1.888671875,
            "mean_seconds": 0.08317016281653195
          },
          "tracking": {
            "accuracy": 0.892578125,
            "mean_repeats": 1.671875,
            "mean_seconds": 0.08195951357483863
          }
        }
      },
      "uniform": {
        "accuracy": 0.8349609375,
        "mean_repeats": 1.5830078125,
        "mean_seconds": 0.07535781949118245,
        "by_task": {
          "boolq": {
            "accuracy": 0.80078125,
            "mean_repeats": 1.517578125,
            "mean_seconds": 0.0850049656815827
          },
          "piqa": {
            "accuracy": 0.818359375,
            "mean_repeats": 1.923828125,
            "mean_seconds": 0.08304794593714178
          },
          "arc": {
            "accuracy": 0.810546875,
            "mean_repeats": 1.6484375,
            "mean_seconds": 0.07719410023419185
          },
          "tracking": {
            "accuracy": 0.91015625,
            "mean_repeats": 1.2421875,
            "mean_seconds": 0.056184266111813486
          }
        }
      },
      "early-weighted": {
        "accuracy": 0.84521484375,
        "mean_repeats": 1.72802734375,
        "mean_seconds": 0.0783738942380296,
        "by_task": {
          "boolq": {
            "accuracy": 0.82421875,
            "mean_repeats": 1.759765625,
            "mean_seconds": 0.09569437745958567
          },
          "piqa": {
            "accuracy": 0.82421875,
            "mean_repeats": 2.169921875,
            "mean_seconds": 0.08601552160689607
          },
          "arc": {
            "accuracy": 0.81640625,
            "mean_repeats": 1.72265625,
            "mean_seconds": 0.07708502726745792
          },
          "tracking": {
            "accuracy": 0.916015625,
            "mean_repeats": 1.259765625,
            "mean_seconds": 0.05470065061817876
          }
        }
      },
      "primary_point_pass": false
    },
    "20260905": {
      "unchanged": {
        "accuracy": 0.771484375,
        "mean_repeats": 3.076171875,
        "mean_seconds": 0.14676617159711897,
        "by_task": {
          "boolq": {
            "accuracy": 0.8125,
            "mean_repeats": 3.21875,
            "mean_seconds": 0.15500486951204948
          },
          "piqa": {
            "accuracy": 0.693359375,
            "mean_repeats": 3.474609375,
            "mean_seconds": 0.15841019880608656
          },
          "arc": {
            "accuracy": 0.830078125,
            "mean_repeats": 2.68359375,
            "mean_seconds": 0.1327572459471412
          },
          "tracking": {
            "accuracy": 0.75,
            "mean_repeats": 2.927734375,
            "mean_seconds": 0.14089237212319858
          }
        }
      },
      "unchanged-fixed4": {
        "accuracy": 0.779296875,
        "mean_repeats": 4,
        "mean_seconds": 0.1962719114686479,
        "by_task": {
          "boolq": {
            "accuracy": 0.818359375,
            "mean_repeats": 4,
            "mean_seconds": 0.1970482659700792
          },
          "piqa": {
            "accuracy": 0.6953125,
            "mean_repeats": 4,
            "mean_seconds": 0.1956732853257563
          },
          "arc": {
            "accuracy": 0.849609375,
            "mean_repeats": 4,
            "mean_seconds": 0.1961194892122876
          },
          "tracking": {
            "accuracy": 0.75390625,
            "mean_repeats": 4,
            "mean_seconds": 0.19624660536646843
          }
        }
      },
      "final-only": {
        "accuracy": 0.83349609375,
        "mean_repeats": 2.1025390625,
        "mean_seconds": 0.1022195853642188,
        "by_task": {
          "boolq": {
            "accuracy": 0.845703125,
            "mean_repeats": 2.30859375,
            "mean_seconds": 0.12289498038007876
          },
          "piqa": {
            "accuracy": 0.798828125,
            "mean_repeats": 2.412109375,
            "mean_seconds": 0.10373532625380903
          },
          "arc": {
            "accuracy": 0.798828125,
            "mean_repeats": 1.9375,
            "mean_seconds": 0.0902546587632969
          },
          "tracking": {
            "accuracy": 0.890625,
            "mean_repeats": 1.751953125,
            "mean_seconds": 0.09199337605969049
          }
        }
      },
      "uniform": {
        "accuracy": 0.85498046875,
        "mean_repeats": 1.734375,
        "mean_seconds": 0.08463800038298358,
        "by_task": {
          "boolq": {
            "accuracy": 0.83984375,
            "mean_repeats": 1.81640625,
            "mean_seconds": 0.10606640282203444
          },
          "piqa": {
            "accuracy": 0.83203125,
            "mean_repeats": 2.107421875,
            "mean_seconds": 0.08090957735548726
          },
          "arc": {
            "accuracy": 0.822265625,
            "mean_repeats": 1.720703125,
            "mean_seconds": 0.08563400942948647
          },
          "tracking": {
            "accuracy": 0.92578125,
            "mean_repeats": 1.29296875,
            "mean_seconds": 0.06594201192492619
          }
        }
      },
      "early-weighted": {
        "accuracy": 0.857421875,
        "mean_repeats": 1.85693359375,
        "mean_seconds": 0.08881599365995499,
        "by_task": {
          "boolq": {
            "accuracy": 0.853515625,
            "mean_repeats": 1.9609375,
            "mean_seconds": 0.1075782822619658
          },
          "piqa": {
            "accuracy": 0.81640625,
            "mean_repeats": 2.341796875,
            "mean_seconds": 0.09304464436136185
          },
          "arc": {
            "accuracy": 0.83203125,
            "mean_repeats": 1.794921875,
            "mean_seconds": 0.09019546213676222
          },
          "tracking": {
            "accuracy": 0.927734375,
            "mean_repeats": 1.330078125,
            "mean_seconds": 0.06444558587973007
          }
        }
      },
      "primary_point_pass": true
    },
    "20260906": {
      "unchanged": {
        "accuracy": 0.779296875,
        "mean_repeats": 3.05419921875,
        "mean_seconds": 0.1415154011323466,
        "by_task": {
          "boolq": {
            "accuracy": 0.80859375,
            "mean_repeats": 3.23046875,
            "mean_seconds": 0.15026386302197353
          },
          "piqa": {
            "accuracy": 0.720703125,
            "mean_repeats": 3.599609375,
            "mean_seconds": 0.16300545127596705
          },
          "arc": {
            "accuracy": 0.828125,
            "mean_repeats": 2.765625,
            "mean_seconds": 0.13321619511116295
          },
          "tracking": {
            "accuracy": 0.759765625,
            "mean_repeats": 2.62109375,
            "mean_seconds": 0.11957609512028285
          }
        }
      },
      "unchanged-fixed4": {
        "accuracy": 0.78515625,
        "mean_repeats": 4,
        "mean_seconds": 0.19428460859635377,
        "by_task": {
          "boolq": {
            "accuracy": 0.8125,
            "mean_repeats": 4,
            "mean_seconds": 0.19482919032452628
          },
          "piqa": {
            "accuracy": 0.720703125,
            "mean_repeats": 4,
            "mean_seconds": 0.19389435035409402
          },
          "arc": {
            "accuracy": 0.845703125,
            "mean_repeats": 4,
            "mean_seconds": 0.1941781899658963
          },
          "tracking": {
            "accuracy": 0.76171875,
            "mean_repeats": 4,
            "mean_seconds": 0.19423670374089852
          }
        }
      },
      "final-only": {
        "accuracy": 0.8583984375,
        "mean_repeats": 2.501953125,
        "mean_seconds": 0.12192506549617974,
        "by_task": {
          "boolq": {
            "accuracy": 0.888671875,
            "mean_repeats": 2.78515625,
            "mean_seconds": 0.13216247597592884
          },
          "piqa": {
            "accuracy": 0.80859375,
            "mean_repeats": 3.0078125,
            "mean_seconds": 0.1388156774570234
          },
          "arc": {
            "accuracy": 0.83203125,
            "mean_repeats": 2.166015625,
            "mean_seconds": 0.104508215514943
          },
          "tracking": {
            "accuracy": 0.904296875,
            "mean_repeats": 2.048828125,
            "mean_seconds": 0.11221389303682372
          }
        }
      },
      "uniform": {
        "accuracy": 0.8603515625,
        "mean_repeats": 1.7294921875,
        "mean_seconds": 0.07998383540398207,
        "by_task": {
          "boolq": {
            "accuracy": 0.8515625,
            "mean_repeats": 1.689453125,
            "mean_seconds": 0.08369938011746852
          },
          "piqa": {
            "accuracy": 0.82421875,
            "mean_repeats": 2.103515625,
            "mean_seconds": 0.09069722979329527
          },
          "arc": {
            "accuracy": 0.828125,
            "mean_repeats": 1.75,
            "mean_seconds": 0.08026932471548207
          },
          "tracking": {
            "accuracy": 0.9375,
            "mean_repeats": 1.375,
            "mean_seconds": 0.06526940698968248
          }
        }
      },
      "early-weighted": {
        "accuracy": 0.8525390625,
        "mean_repeats": 1.78173828125,
        "mean_seconds": 0.08119497742009116,
        "by_task": {
          "boolq": {
            "accuracy": 0.849609375,
            "mean_repeats": 1.91015625,
            "mean_seconds": 0.09897847615648062
          },
          "piqa": {
            "accuracy": 0.806640625,
            "mean_repeats": 2.19140625,
            "mean_seconds": 0.08622411096584984
          },
          "arc": {
            "accuracy": 0.818359375,
            "mean_repeats": 1.712890625,
            "mean_seconds": 0.08182545389863663
          },
          "tracking": {
            "accuracy": 0.935546875,
            "mean_repeats": 1.3125,
            "mean_seconds": 0.05775186865939759
          }
        }
      },
      "primary_point_pass": true
    }
  },
  "validation": {
    "20260904": {
      "unchanged": {
        "questions": 512,
        "accuracy_by_repeat": [
          0.5859375,
          0.724609375,
          0.775390625,
          0.7890625
        ],
        "selected_accuracy": 0.787109375,
        "mean_repeats": 3.150390625,
        "early_stops": 239,
        "early_wrong_final_correct": 1,
        "early_correct_final_wrong": 0,
        "by_task": {
          "boolq": {
            "accuracy_by_repeat": [
              0.5234375,
              0.6171875,
              0.7265625,
              0.75
            ],
            "mean_repeats": 3.15625
          },
          "piqa": {
            "accuracy_by_repeat": [
              0.640625,
              0.7265625,
              0.7421875,
              0.75
            ],
            "mean_repeats": 3.578125
          },
          "arc": {
            "accuracy_by_repeat": [
              0.5234375,
              0.8359375,
              0.8828125,
              0.90625
            ],
            "mean_repeats": 2.7578125
          },
          "tracking": {
            "accuracy_by_repeat": [
              0.65625,
              0.71875,
              0.75,
              0.75
            ],
            "mean_repeats": 3.109375
          }
        }
      },
      "final-only": {
        "questions": 512,
        "accuracy_by_repeat": [
          0.6484375,
          0.84375,
          0.900390625,
          0.896484375
        ],
        "selected_accuracy": 0.853515625,
        "mean_repeats": 1.955078125,
        "early_stops": 472,
        "early_wrong_final_correct": 27,
        "early_correct_final_wrong": 5,
        "by_task": {
          "boolq": {
            "accuracy_by_repeat": [
              0.625,
              0.8125,
              0.8828125,
              0.875
            ],
            "mean_repeats": 2.0390625
          },
          "piqa": {
            "accuracy_by_repeat": [
              0.71875,
              0.8359375,
              0.8359375,
              0.8125
            ],
            "mean_repeats": 2.28125
          },
          "arc": {
            "accuracy_by_repeat": [
              0.546875,
              0.859375,
              0.953125,
              0.9453125
            ],
            "mean_repeats": 1.8984375
          },
          "tracking": {
            "accuracy_by_repeat": [
              0.703125,
              0.8671875,
              0.9296875,
              0.953125
            ],
            "mean_repeats": 1.6015625
          }
        }
      },
      "uniform": {
        "questions": 512,
        "accuracy_by_repeat": [
          0.71875,
          0.87890625,
          0.90625,
          0.8984375
        ],
        "selected_accuracy": 0.8515625,
        "mean_repeats": 1.541015625,
        "early_stops": 497,
        "early_wrong_final_correct": 33,
        "early_correct_final_wrong": 9,
        "by_task": {
          "boolq": {
            "accuracy_by_repeat": [
              0.71875,
              0.84375,
              0.875,
              0.8515625
            ],
            "mean_repeats": 1.53125
          },
          "piqa": {
            "accuracy_by_repeat": [
              0.703125,
              0.8203125,
              0.8359375,
              0.8125
            ],
            "mean_repeats": 1.84375
          },
          "arc": {
            "accuracy_by_repeat": [
              0.6484375,
              0.8984375,
              0.953125,
              0.96875
            ],
            "mean_repeats": 1.5390625
          },
          "tracking": {
            "accuracy_by_repeat": [
              0.8046875,
              0.953125,
              0.9609375,
              0.9609375
            ],
            "mean_repeats": 1.25
          }
        }
      },
      "early-weighted": {
        "questions": 512,
        "accuracy_by_repeat": [
          0.73828125,
          0.88671875,
          0.90625,
          0.908203125
        ],
        "selected_accuracy": 0.869140625,
        "mean_repeats": 1.71875,
        "early_stops": 465,
        "early_wrong_final_correct": 24,
        "early_correct_final_wrong": 4,
        "by_task": {
          "boolq": {
            "accuracy_by_repeat": [
              0.734375,
              0.859375,
              0.875,
              0.875
            ],
            "mean_repeats": 1.8828125
          },
          "piqa": {
            "accuracy_by_repeat": [
              0.7109375,
              0.828125,
              0.84375,
              0.828125
            ],
            "mean_repeats": 2.125
          },
          "arc": {
            "accuracy_by_repeat": [
              0.671875,
              0.90625,
              0.9609375,
              0.96875
            ],
            "mean_repeats": 1.640625
          },
          "tracking": {
            "accuracy_by_repeat": [
              0.8359375,
              0.953125,
              0.9453125,
              0.9609375
            ],
            "mean_repeats": 1.2265625
          }
        }
      }
    },
    "20260905": {
      "unchanged": {
        "questions": 512,
        "accuracy_by_repeat": [
          0.5703125,
          0.712890625,
          0.794921875,
          0.796875
        ],
        "selected_accuracy": 0.79296875,
        "mean_repeats": 3.087890625,
        "early_stops": 254,
        "early_wrong_final_correct": 2,
        "early_correct_final_wrong": 0,
        "by_task": {
          "boolq": {
            "accuracy_by_repeat": [
              0.5625,
              0.6328125,
              0.7890625,
              0.7890625
            ],
            "mean_repeats": 3.1953125
          },
          "piqa": {
            "accuracy_by_repeat": [
              0.6015625,
              0.6953125,
              0.75,
              0.7421875
            ],
            "mean_repeats": 3.46875
          },
          "arc": {
            "accuracy_by_repeat": [
              0.5,
              0.828125,
              0.8984375,
              0.8984375
            ],
            "mean_repeats": 2.75
          },
          "tracking": {
            "accuracy_by_repeat": [
              0.6171875,
              0.6953125,
              0.7421875,
              0.7578125
            ],
            "mean_repeats": 2.9375
          }
        }
      },
      "final-only": {
        "questions": 512,
        "accuracy_by_repeat": [
          0.611328125,
          0.833984375,
          0.896484375,
          0.90625
        ],
        "selected_accuracy": 0.861328125,
        "mean_repeats": 2.0703125,
        "early_stops": 448,
        "early_wrong_final_correct": 28,
        "early_correct_final_wrong": 5,
        "by_task": {
          "boolq": {
            "accuracy_by_repeat": [
              0.6796875,
              0.8125,
              0.875,
              0.875
            ],
            "mean_repeats": 2.3359375
          },
          "piqa": {
            "accuracy_by_repeat": [
              0.609375,
              0.765625,
              0.828125,
              0.8359375
            ],
            "mean_repeats": 2.359375
          },
          "arc": {
            "accuracy_by_repeat": [
              0.4765625,
              0.8671875,
              0.9375,
              0.96875
            ],
            "mean_repeats": 1.9453125
          },
          "tracking": {
            "accuracy_by_repeat": [
              0.6796875,
              0.890625,
              0.9453125,
              0.9453125
            ],
            "mean_repeats": 1.640625
          }
        }
      },
      "uniform": {
        "questions": 512,
        "accuracy_by_repeat": [
          0.724609375,
          0.884765625,
          0.8984375,
          0.90625
        ],
        "selected_accuracy": 0.86328125,
        "mean_repeats": 1.6640625,
        "early_stops": 481,
        "early_wrong_final_correct": 25,
        "early_correct_final_wrong": 3,
        "by_task": {
          "boolq": {
            "accuracy_by_repeat": [
              0.7421875,
              0.859375,
              0.8828125,
              0.859375
            ],
            "mean_repeats": 1.703125
          },
          "piqa": {
            "accuracy_by_repeat": [
              0.6640625,
              0.84375,
              0.828125,
              0.859375
            ],
            "mean_repeats": 2.0390625
          },
          "arc": {
            "accuracy_by_repeat": [
              0.6484375,
              0.890625,
              0.9375,
              0.9609375
            ],
            "mean_repeats": 1.6796875
          },
          "tracking": {
            "accuracy_by_repeat": [
              0.84375,
              0.9453125,
              0.9453125,
              0.9453125
            ],
            "mean_repeats": 1.234375
          }
        }
      },
      "early-weighted": {
        "questions": 512,
        "accuracy_by_repeat": [
          0.75390625,
          0.884765625,
          0.89453125,
          0.90625
        ],
        "selected_accuracy": 0.87109375,
        "mean_repeats": 1.759765625,
        "early_stops": 462,
        "early_wrong_final_correct": 20,
        "early_correct_final_wrong": 2,
        "by_task": {
          "boolq": {
            "accuracy_by_repeat": [
              0.765625,
              0.859375,
              0.859375,
              0.859375
            ],
            "mean_repeats": 1.859375
          },
          "piqa": {
            "accuracy_by_repeat": [
              0.703125,
              0.8515625,
              0.84375,
              0.8515625
            ],
            "mean_repeats": 2.21875
          },
          "arc": {
            "accuracy_by_repeat": [
              0.671875,
              0.890625,
              0.9375,
              0.9609375
            ],
            "mean_repeats": 1.75
          },
          "tracking": {
            "accuracy_by_repeat": [
              0.875,
              0.9375,
              0.9375,
              0.953125
            ],
            "mean_repeats": 1.2109375
          }
        }
      }
    },
    "20260906": {
      "unchanged": {
        "questions": 512,
        "accuracy_by_repeat": [
          0.603515625,
          0.703125,
          0.796875,
          0.8046875
        ],
        "selected_accuracy": 0.80078125,
        "mean_repeats": 2.998046875,
        "early_stops": 269,
        "early_wrong_final_correct": 2,
        "early_correct_final_wrong": 0,
        "by_task": {
          "boolq": {
            "accuracy_by_repeat": [
              0.546875,
              0.5703125,
              0.7578125,
              0.796875
            ],
            "mean_repeats": 3.1484375
          },
          "piqa": {
            "accuracy_by_repeat": [
              0.65625,
              0.6875,
              0.78125,
              0.7578125
            ],
            "mean_repeats": 3.5078125
          },
          "arc": {
            "accuracy_by_repeat": [
              0.5546875,
              0.828125,
              0.8828125,
              0.890625
            ],
            "mean_repeats": 2.7578125
          },
          "tracking": {
            "accuracy_by_repeat": [
              0.65625,
              0.7265625,
              0.765625,
              0.7734375
            ],
            "mean_repeats": 2.578125
          }
        }
      },
      "final-only": {
        "questions": 512,
        "accuracy_by_repeat": [
          0.603515625,
          0.828125,
          0.88671875,
          0.90234375
        ],
        "selected_accuracy": 0.87890625,
        "mean_repeats": 2.48046875,
        "early_stops": 390,
        "early_wrong_final_correct": 14,
        "early_correct_final_wrong": 2,
        "by_task": {
          "boolq": {
            "accuracy_by_repeat": [
              0.6015625,
              0.765625,
              0.8671875,
              0.875
            ],
            "mean_repeats": 2.765625
          },
          "piqa": {
            "accuracy_by_repeat": [
              0.6328125,
              0.8359375,
              0.84375,
              0.8515625
            ],
            "mean_repeats": 2.859375
          },
          "arc": {
            "accuracy_by_repeat": [
              0.5390625,
              0.8671875,
              0.9296875,
              0.9375
            ],
            "mean_repeats": 2.1796875
          },
          "tracking": {
            "accuracy_by_repeat": [
              0.640625,
              0.84375,
              0.90625,
              0.9453125
            ],
            "mean_repeats": 2.1171875
          }
        }
      },
      "uniform": {
        "questions": 512,
        "accuracy_by_repeat": [
          0.73828125,
          0.884765625,
          0.91015625,
          0.912109375
        ],
        "selected_accuracy": 0.865234375,
        "mean_repeats": 1.666015625,
        "early_stops": 481,
        "early_wrong_final_correct": 27,
        "early_correct_final_wrong": 3,
        "by_task": {
          "boolq": {
            "accuracy_by_repeat": [
              0.78125,
              0.875,
              0.8828125,
              0.875
            ],
            "mean_repeats": 1.59375
          },
          "piqa": {
            "accuracy_by_repeat": [
              0.71875,
              0.8359375,
              0.8515625,
              0.8515625
            ],
            "mean_repeats": 2.0859375
          },
          "arc": {
            "accuracy_by_repeat": [
              0.625,
              0.8828125,
              0.9453125,
              0.9609375
            ],
            "mean_repeats": 1.6953125
          },
          "tracking": {
            "accuracy_by_repeat": [
              0.828125,
              0.9453125,
              0.9609375,
              0.9609375
            ],
            "mean_repeats": 1.2890625
          }
        }
      },
      "early-weighted": {
        "questions": 512,
        "accuracy_by_repeat": [
          0.74609375,
          0.88671875,
          0.91015625,
          0.912109375
        ],
        "selected_accuracy": 0.876953125,
        "mean_repeats": 1.724609375,
        "early_stops": 463,
        "early_wrong_final_correct": 22,
        "early_correct_final_wrong": 4,
        "by_task": {
          "boolq": {
            "accuracy_by_repeat": [
              0.7890625,
              0.875,
              0.8828125,
              0.859375
            ],
            "mean_repeats": 1.8203125
          },
          "piqa": {
            "accuracy_by_repeat": [
              0.7109375,
              0.8359375,
              0.8515625,
              0.859375
            ],
            "mean_repeats": 2.125
          },
          "arc": {
            "accuracy_by_repeat": [
              0.6328125,
              0.890625,
              0.9453125,
              0.9609375
            ],
            "mean_repeats": 1.671875
          },
          "tracking": {
            "accuracy_by_repeat": [
              0.8515625,
              0.9453125,
              0.9609375,
              0.96875
            ],
            "mean_repeats": 1.28125
          }
        }
      }
    }
  },
  "scope": "Batch-one, no-cache, short-answer generation. Intervals condition on three trained models and the sampled workload.",
  "protocol_sha256": "3670d499c45909a97ba8d068745bf100248ed9f213ddef387995ce84697fff8e",
  "model": "Ouro-1.4B",
  "train_questions": 2048,
  "validation_questions": 512,
  "test_questions": 2048,
  "timing_questions": 128,
  "timing_rounds": 5,
  "verification": {
    "all_checks_passed": true,
    "counts": {
      "optimizer_updates": 2304,
      "fixed_validation_outputs": 24576,
      "calibrated_question_decisions": 1241088,
      "actual_test_generations": 30720,
      "actual_timing_replays": 9600
    }
  },
  "rows_sha256": "8b4073a5d76231b2fdbf0ea90bb753713f71ac3262535ed2db280d322d191515",
  "calibration": {
    "20260904": {
      "unchanged": {
        "selected": {
          "threshold": 0.905,
          "accuracy": 0.787109375,
          "mean_repeats": 3.150390625,
          "by_task": {
            "boolq": {
              "accuracy": 0.75,
              "repeats": 3.15625,
              "reference_accuracy": 0.75
            },
            "piqa": {
              "accuracy": 0.75,
              "repeats": 3.578125,
              "reference_accuracy": 0.75
            },
            "arc": {
              "accuracy": 0.8984375,
              "repeats": 2.7578125,
              "reference_accuracy": 0.90625
            },
            "tracking": {
              "accuracy": 0.75,
              "repeats": 3.109375,
              "reference_accuracy": 0.75
            }
          },
          "feasible": true
        },
        "feasible": true
      },
      "final-only": {
        "selected": {
          "threshold": 0.825,
          "accuracy": 0.853515625,
          "mean_repeats": 1.955078125,
          "by_task": {
            "boolq": {
              "accuracy": 0.84375,
              "repeats": 2.0390625,
              "reference_accuracy": 0.75
            },
            "piqa": {
              "accuracy": 0.828125,
              "repeats": 2.28125,
              "reference_accuracy": 0.75
            },
            "arc": {
              "accuracy": 0.8984375,
              "repeats": 1.8984375,
              "reference_accuracy": 0.90625
            },
            "tracking": {
              "accuracy": 0.84375,
              "repeats": 1.6015625,
              "reference_accuracy": 0.75
            }
          },
          "feasible": true
        },
        "feasible": true
      },
      "uniform": {
        "selected": {
          "threshold": 0.74,
          "accuracy": 0.8515625,
          "mean_repeats": 1.541015625,
          "by_task": {
            "boolq": {
              "accuracy": 0.8046875,
              "repeats": 1.53125,
              "reference_accuracy": 0.75
            },
            "piqa": {
              "accuracy": 0.8125,
              "repeats": 1.84375,
              "reference_accuracy": 0.75
            },
            "arc": {
              "accuracy": 0.8984375,
              "repeats": 1.5390625,
              "reference_accuracy": 0.90625
            },
            "tracking": {
              "accuracy": 0.890625,
              "repeats": 1.25,
              "reference_accuracy": 0.75
            }
          },
          "feasible": true
        },
        "feasible": true
      },
      "early-weighted": {
        "selected": {
          "threshold": 0.84,
          "accuracy": 0.869140625,
          "mean_repeats": 1.71875,
          "by_task": {
            "boolq": {
              "accuracy": 0.8515625,
              "repeats": 1.8828125,
              "reference_accuracy": 0.75
            },
            "piqa": {
              "accuracy": 0.8125,
              "repeats": 2.125,
              "reference_accuracy": 0.75
            },
            "arc": {
              "accuracy": 0.90625,
              "repeats": 1.640625,
              "reference_accuracy": 0.90625
            },
            "tracking": {
              "accuracy": 0.90625,
              "repeats": 1.2265625,
              "reference_accuracy": 0.75
            }
          },
          "feasible": true
        },
        "feasible": true
      }
    },
    "20260905": {
      "unchanged": {
        "selected": {
          "threshold": 0.84,
          "accuracy": 0.79296875,
          "mean_repeats": 3.087890625,
          "by_task": {
            "boolq": {
              "accuracy": 0.78125,
              "repeats": 3.1953125,
              "reference_accuracy": 0.7890625
            },
            "piqa": {
              "accuracy": 0.7421875,
              "repeats": 3.46875,
              "reference_accuracy": 0.7421875
            },
            "arc": {
              "accuracy": 0.890625,
              "repeats": 2.75,
              "reference_accuracy": 0.8984375
            },
            "tracking": {
              "accuracy": 0.7578125,
              "repeats": 2.9375,
              "reference_accuracy": 0.7578125
            }
          },
          "feasible": true
        },
        "feasible": true
      },
      "final-only": {
        "selected": {
          "threshold": 0.86,
          "accuracy": 0.861328125,
          "mean_repeats": 2.0703125,
          "by_task": {
            "boolq": {
              "accuracy": 0.875,
              "repeats": 2.3359375,
              "reference_accuracy": 0.7890625
            },
            "piqa": {
              "accuracy": 0.8359375,
              "repeats": 2.359375,
              "reference_accuracy": 0.7421875
            },
            "arc": {
              "accuracy": 0.890625,
              "repeats": 1.9453125,
              "reference_accuracy": 0.8984375
            },
            "tracking": {
              "accuracy": 0.84375,
              "repeats": 1.640625,
              "reference_accuracy": 0.7578125
            }
          },
          "feasible": true
        },
        "feasible": true
      },
      "uniform": {
        "selected": {
          "threshold": 0.78,
          "accuracy": 0.86328125,
          "mean_repeats": 1.6640625,
          "by_task": {
            "boolq": {
              "accuracy": 0.828125,
              "repeats": 1.703125,
              "reference_accuracy": 0.7890625
            },
            "piqa": {
              "accuracy": 0.828125,
              "repeats": 2.0390625,
              "reference_accuracy": 0.7421875
            },
            "arc": {
              "accuracy": 0.890625,
              "repeats": 1.6796875,
              "reference_accuracy": 0.8984375
            },
            "tracking": {
              "accuracy": 0.90625,
              "repeats": 1.234375,
              "reference_accuracy": 0.7578125
            }
          },
          "feasible": true
        },
        "feasible": true
      },
      "early-weighted": {
        "selected": {
          "threshold": 0.84,
          "accuracy": 0.87109375,
          "mean_repeats": 1.759765625,
          "by_task": {
            "boolq": {
              "accuracy": 0.8515625,
              "repeats": 1.859375,
              "reference_accuracy": 0.7890625
            },
            "piqa": {
              "accuracy": 0.8203125,
              "repeats": 2.21875,
              "reference_accuracy": 0.7421875
            },
            "arc": {
              "accuracy": 0.890625,
              "repeats": 1.75,
              "reference_accuracy": 0.8984375
            },
            "tracking": {
              "accuracy": 0.921875,
              "repeats": 1.2109375,
              "reference_accuracy": 0.7578125
            }
          },
          "feasible": true
        },
        "feasible": true
      }
    },
    "20260906": {
      "unchanged": {
        "selected": {
          "threshold": 0.915,
          "accuracy": 0.80078125,
          "mean_repeats": 2.998046875,
          "by_task": {
            "boolq": {
              "accuracy": 0.796875,
              "repeats": 3.1484375,
              "reference_accuracy": 0.796875
            },
            "piqa": {
              "accuracy": 0.7578125,
              "repeats": 3.5078125,
              "reference_accuracy": 0.7578125
            },
            "arc": {
              "accuracy": 0.8828125,
              "repeats": 2.7578125,
              "reference_accuracy": 0.890625
            },
            "tracking": {
              "accuracy": 0.765625,
              "repeats": 2.578125,
              "reference_accuracy": 0.7734375
            }
          },
          "feasible": true
        },
        "feasible": true
      },
      "final-only": {
        "selected": {
          "threshold": 0.97,
          "accuracy": 0.87890625,
          "mean_repeats": 2.48046875,
          "by_task": {
            "boolq": {
              "accuracy": 0.875,
              "repeats": 2.765625,
              "reference_accuracy": 0.796875
            },
            "piqa": {
              "accuracy": 0.8515625,
              "repeats": 2.859375,
              "reference_accuracy": 0.7578125
            },
            "arc": {
              "accuracy": 0.8828125,
              "repeats": 2.1796875,
              "reference_accuracy": 0.890625
            },
            "tracking": {
              "accuracy": 0.90625,
              "repeats": 2.1171875,
              "reference_accuracy": 0.7734375
            }
          },
          "feasible": true
        },
        "feasible": true
      },
      "uniform": {
        "selected": {
          "threshold": 0.815,
          "accuracy": 0.865234375,
          "mean_repeats": 1.666015625,
          "by_task": {
            "boolq": {
              "accuracy": 0.828125,
              "repeats": 1.59375,
              "reference_accuracy": 0.796875
            },
            "piqa": {
              "accuracy": 0.8359375,
              "repeats": 2.0859375,
              "reference_accuracy": 0.7578125
            },
            "arc": {
              "accuracy": 0.8828125,
              "repeats": 1.6953125,
              "reference_accuracy": 0.890625
            },
            "tracking": {
              "accuracy": 0.9140625,
              "repeats": 1.2890625,
              "reference_accuracy": 0.7734375
            }
          },
          "feasible": true
        },
        "feasible": true
      },
      "early-weighted": {
        "selected": {
          "threshold": 0.83,
          "accuracy": 0.876953125,
          "mean_repeats": 1.724609375,
          "by_task": {
            "boolq": {
              "accuracy": 0.8515625,
              "repeats": 1.8203125,
              "reference_accuracy": 0.796875
            },
            "piqa": {
              "accuracy": 0.84375,
              "repeats": 2.125,
              "reference_accuracy": 0.7578125
            },
            "arc": {
              "accuracy": 0.890625,
              "repeats": 1.671875,
              "reference_accuracy": 0.890625
            },
            "tracking": {
              "accuracy": 0.921875,
              "repeats": 1.28125,
              "reference_accuracy": 0.7734375
            }
          },
          "feasible": true
        },
        "feasible": true
      }
    }
  }
}