{ "displacement": { "createdUTC": "2026-10-05T12:12:41.654926+00:00", "status": "Public R02 benchmark, single prespecified exploratory pool reranker; no fresh holdout, no confirmatory p-values.", "methodFreezeSHA256": "e9a333dfccf54ac817f9865772b1528f5ea2e20289c232e6c7e32d6e33621825", "qrelsSHA256": "078c72c16df0dd97cf1ea409d38a22a78940236adc7e50b5c54666083eace325", "historicalTypedSetting": "typed:dense:1:0.1", "oracleWarning": "Label-dependent exact coverage upper bound only; not deployable, not a baseline.", "subsets": { "all48": { "n": 48, "means": { "dense": { "ndcg5": 0.6926657586842738, "agreedObligationCoverage5": 0.7395833333333334, "labelRedundancy": 0.3734177859177859, "embeddingRedundancy": 0.3936426223394245, "documentsWithoutAgreedObligations": 1.9166666666666667, "lostCountVsDense": 0.0, "gainedCountVsDense": 0.0 }, "typed": { "ndcg5": 0.6894028159545785, "agreedObligationCoverage5": 0.734375, "labelRedundancy": 0.3853225478225479, "embeddingRedundancy": 0.39355339724122046, "documentsWithoutAgreedObligations": 1.9166666666666667, "lostCountVsDense": 0.020833333333333332, "gainedCountVsDense": 0.0 }, "coverage": { "ndcg5": 0.6048265188871251, "agreedObligationCoverage5": 0.765625, "labelRedundancy": 0.26879960317460316, "embeddingRedundancy": 0.1839134927337265, "documentsWithoutAgreedObligations": 2.5416666666666665, "lostCountVsDense": 0.25, "gainedCountVsDense": 0.3541666666666667 }, "oracle": { "ndcg5": 0.997849329644325, "agreedObligationCoverage5": 0.8854166666666666, "labelRedundancy": 0.53003848003848, "embeddingRedundancy": 0.35242273837859533, "documentsWithoutAgreedObligations": 0.4791666666666667, "lostCountVsDense": 0.0, "gainedCountVsDense": 0.5833333333333334 } }, "movementVsDense": { "dense": { "uniqueObligationsLostTotal": 0, "uniqueObligationsGainedTotal": 0, "tasksLosingAnyObligation": 0, "tasksGainingAnyObligation": 0 }, "typed": { "uniqueObligationsLostTotal": 1, "uniqueObligationsGainedTotal": 0, "tasksLosingAnyObligation": 1, "tasksGainingAnyObligation": 0 }, "coverage": { "uniqueObligationsLostTotal": 12, "uniqueObligationsGainedTotal": 17, "tasksLosingAnyObligation": 11, "tasksGainingAnyObligation": 13 }, "oracle": { "uniqueObligationsLostTotal": 0, "uniqueObligationsGainedTotal": 28, "tasksLosingAnyObligation": 0, "tasksGainingAnyObligation": 20 } }, "pairedExploratoryEffects": { "coverage-minus-dense": { "ndcg5": { "n": 48, "meanDifference": -0.08783923979714862, "percentile95": [ -0.14072528008888086, -0.035359403065076116 ], "wins": 16, "losses": 32, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "agreedObligationCoverage5": { "n": 48, "meanDifference": 0.026041666666666668, "percentile95": [ -0.036458333333333336, 0.09375 ], "wins": 11, "losses": 9, "ties": 28, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "labelRedundancy": { "n": 48, "meanDifference": -0.10461818274318274, "percentile95": [ -0.17290534696784698, -0.03733718393874651 ], "wins": 10, "losses": 28, "ties": 10, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "embeddingRedundancy": { "n": 48, "meanDifference": -0.20972912960569792, "percentile95": [ -0.2325573819455241, -0.18688125850667278 ], "wins": 0, "losses": 48, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." } }, "coverage-minus-typed": { "ndcg5": { "n": 48, "meanDifference": -0.0845762970674534, "percentile95": [ -0.13872676285506907, -0.031046097616357134 ], "wins": 16, "losses": 32, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "agreedObligationCoverage5": { "n": 48, "meanDifference": 0.03125, "percentile95": [ -0.03125, 0.09895833333333333 ], "wins": 12, "losses": 9, "ties": 27, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "labelRedundancy": { "n": 48, "meanDifference": -0.11652294464794462, "percentile95": [ -0.18607920291514038, -0.04666832010582011 ], "wins": 10, "losses": 29, "ties": 9, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "embeddingRedundancy": { "n": 48, "meanDifference": -0.209639904507494, "percentile95": [ -0.23192405852350625, -0.18705612053816478 ], "wins": 0, "losses": 48, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." } }, "typed-minus-dense": { "ndcg5": { "n": 48, "meanDifference": -0.0032629427296952, "percentile95": [ -0.009077156617727782, 0.0 ], "wins": 0, "losses": 2, "ties": 46, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "agreedObligationCoverage5": { "n": 48, "meanDifference": -0.005208333333333333, "percentile95": [ -0.015625, 0.0 ], "wins": 0, "losses": 1, "ties": 47, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "labelRedundancy": { "n": 48, "meanDifference": 0.011904761904761904, "percentile95": [ 0.0, 0.03422619047619047 ], "wins": 2, "losses": 0, "ties": 46, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "embeddingRedundancy": { "n": 48, "meanDifference": -8.92250982039514e-05, "percentile95": [ -0.003340645670091409, 0.003072970375479555 ], "wins": 1, "losses": 1, "ties": 46, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." } }, "oracle-minus-dense": { "ndcg5": { "n": 48, "meanDifference": 0.30518357096005133, "percentile95": [ 0.253598169493844, 0.35672943943295105 ], "wins": 47, "losses": 0, "ties": 1, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "agreedObligationCoverage5": { "n": 48, "meanDifference": 0.14583333333333334, "percentile95": [ 0.08854166666666667, 0.20833333333333334 ], "wins": 20, "losses": 0, "ties": 28, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "labelRedundancy": { "n": 48, "meanDifference": 0.15662069412069413, "percentile95": [ 0.10134837384837385, 0.21627459924334924 ], "wins": 33, "losses": 7, "ties": 8, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "embeddingRedundancy": { "n": 48, "meanDifference": -0.041219883960829116, "percentile95": [ -0.06303519592389803, -0.020201481350691712 ], "wins": 13, "losses": 35, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." } } }, "poolMissingObligations": 22, "oracleMissesDueToBudget": 0 }, "originalFinal36": { "n": 36, "means": { "dense": { "ndcg5": 0.6740797549180285, "agreedObligationCoverage5": 0.7361111111111112, "labelRedundancy": 0.34854497354497355, "embeddingRedundancy": 0.38734800938097586, "documentsWithoutAgreedObligations": 2.0, "lostCountVsDense": 0.0, "gainedCountVsDense": 0.0 }, "typed": { "ndcg5": 0.6706780600402453, "agreedObligationCoverage5": 0.7291666666666666, "labelRedundancy": 0.36243386243386244, "embeddingRedundancy": 0.3887137739923001, "documentsWithoutAgreedObligations": 2.0, "lostCountVsDense": 0.027777777777777776, "gainedCountVsDense": 0.0 }, "coverage": { "ndcg5": 0.593081387157406, "agreedObligationCoverage5": 0.7291666666666666, "labelRedundancy": 0.2799603174603174, "embeddingRedundancy": 0.1846386888376158, "documentsWithoutAgreedObligations": 2.5555555555555554, "lostCountVsDense": 0.2777777777777778, "gainedCountVsDense": 0.25 }, "oracle": { "ndcg5": 0.9971324395257667, "agreedObligationCoverage5": 0.875, "labelRedundancy": 0.5185175164341831, "embeddingRedundancy": 0.3516551779283255, "documentsWithoutAgreedObligations": 0.5277777777777778, "lostCountVsDense": 0.0, "gainedCountVsDense": 0.5555555555555556 } }, "movementVsDense": { "dense": { "uniqueObligationsLostTotal": 0, "uniqueObligationsGainedTotal": 0, "tasksLosingAnyObligation": 0, "tasksGainingAnyObligation": 0 }, "typed": { "uniqueObligationsLostTotal": 1, "uniqueObligationsGainedTotal": 0, "tasksLosingAnyObligation": 1, "tasksGainingAnyObligation": 0 }, "coverage": { "uniqueObligationsLostTotal": 10, "uniqueObligationsGainedTotal": 9, "tasksLosingAnyObligation": 9, "tasksGainingAnyObligation": 9 }, "oracle": { "uniqueObligationsLostTotal": 0, "uniqueObligationsGainedTotal": 20, "tasksLosingAnyObligation": 0, "tasksGainingAnyObligation": 16 } }, "pairedExploratoryEffects": { "coverage-minus-dense": { "ndcg5": { "n": 36, "meanDifference": -0.0809983677606226, "percentile95": [ -0.13721594589602293, -0.025347655009491984 ], "wins": 12, "losses": 24, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "agreedObligationCoverage5": { "n": 36, "meanDifference": -0.006944444444444444, "percentile95": [ -0.0625, 0.04861111111111111 ], "wins": 7, "losses": 7, "ties": 22, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "labelRedundancy": { "n": 36, "meanDifference": -0.06858465608465608, "percentile95": [ -0.1412045304232804, 0.0038707010582010298 ], "wins": 9, "losses": 18, "ties": 9, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "embeddingRedundancy": { "n": 36, "meanDifference": -0.2027093205433601, "percentile95": [ -0.2282716570169672, -0.17820154994596177 ], "wins": 0, "losses": 36, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." } }, "coverage-minus-typed": { "ndcg5": { "n": 36, "meanDifference": -0.07759667288283942, "percentile95": [ -0.13502133443655195, -0.02072633007762613 ], "wins": 12, "losses": 24, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "agreedObligationCoverage5": { "n": 36, "meanDifference": 0.0, "percentile95": [ -0.0557291666666665, 0.05555555555555555 ], "wins": 8, "losses": 7, "ties": 21, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "labelRedundancy": { "n": 36, "meanDifference": -0.08247354497354498, "percentile95": [ -0.15836722883597884, -0.005949074074074119 ], "wins": 9, "losses": 19, "ties": 8, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "embeddingRedundancy": { "n": 36, "meanDifference": -0.20407508515468434, "percentile95": [ -0.22969976630070113, -0.17907249259967448 ], "wins": 0, "losses": 36, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." } }, "typed-minus-dense": { "ndcg5": { "n": 36, "meanDifference": -0.0034016948777831745, "percentile95": [ -0.010205084633349524, 0.0 ], "wins": 0, "losses": 1, "ties": 35, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "agreedObligationCoverage5": { "n": 36, "meanDifference": -0.006944444444444444, "percentile95": [ -0.020833333333333332, 0.0 ], "wins": 0, "losses": 1, "ties": 35, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "labelRedundancy": { "n": 36, "meanDifference": 0.013888888888888888, "percentile95": [ 0.0, 0.041666666666666664 ], "wins": 1, "losses": 0, "ties": 35, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "embeddingRedundancy": { "n": 36, "meanDifference": 0.0013657646113242466, "percentile95": [ 0.0, 0.00409729383397274 ], "wins": 1, "losses": 0, "ties": 35, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." } }, "oracle-minus-dense": { "ndcg5": { "n": 36, "meanDifference": 0.3230526846077381, "percentile95": [ 0.2729066773996696, 0.37496265334381773 ], "wins": 36, "losses": 0, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "agreedObligationCoverage5": { "n": 36, "meanDifference": 0.1388888888888889, "percentile95": [ 0.08333333333333333, 0.2013888888888889 ], "wins": 16, "losses": 0, "ties": 20, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "labelRedundancy": { "n": 36, "meanDifference": 0.16997254288920954, "percentile95": [ 0.11153236231361231, 0.23273511403719735 ], "wins": 27, "losses": 4, "ties": 5, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "embeddingRedundancy": { "n": 36, "meanDifference": -0.03569283145265037, "percentile95": [ -0.059000030945308535, -0.012240420694356267 ], "wins": 10, "losses": 26, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." } } }, "poolMissingObligations": 18, "oracleMissesDueToBudget": 0 }, "originalDevelopment12": { "n": 12, "means": { "dense": { "ndcg5": 0.7484237699830091, "agreedObligationCoverage5": 0.75, "labelRedundancy": 0.448036223036223, "embeddingRedundancy": 0.41252646121477016, "documentsWithoutAgreedObligations": 1.6666666666666667, "lostCountVsDense": 0.0, "gainedCountVsDense": 0.0 }, "typed": { "ndcg5": 0.7455770836975778, "agreedObligationCoverage5": 0.75, "labelRedundancy": 0.4539886039886039, "embeddingRedundancy": 0.40807226698798166, "documentsWithoutAgreedObligations": 1.6666666666666667, "lostCountVsDense": 0.0, "gainedCountVsDense": 0.0 }, "coverage": { "ndcg5": 0.6400619140762824, "agreedObligationCoverage5": 0.875, "labelRedundancy": 0.23531746031746034, "embeddingRedundancy": 0.18173790442205873, "documentsWithoutAgreedObligations": 2.5, "lostCountVsDense": 0.16666666666666666, "gainedCountVsDense": 0.6666666666666666 }, "oracle": { "ndcg5": 1.0, "agreedObligationCoverage5": 0.9166666666666666, "labelRedundancy": 0.5646013708513709, "embeddingRedundancy": 0.3547254197294048, "documentsWithoutAgreedObligations": 0.3333333333333333, "lostCountVsDense": 0.0, "gainedCountVsDense": 0.6666666666666666 } }, "movementVsDense": { "dense": { "uniqueObligationsLostTotal": 0, "uniqueObligationsGainedTotal": 0, "tasksLosingAnyObligation": 0, "tasksGainingAnyObligation": 0 }, "typed": { "uniqueObligationsLostTotal": 0, "uniqueObligationsGainedTotal": 0, "tasksLosingAnyObligation": 0, "tasksGainingAnyObligation": 0 }, "coverage": { "uniqueObligationsLostTotal": 2, "uniqueObligationsGainedTotal": 8, "tasksLosingAnyObligation": 2, "tasksGainingAnyObligation": 4 }, "oracle": { "uniqueObligationsLostTotal": 0, "uniqueObligationsGainedTotal": 8, "tasksLosingAnyObligation": 0, "tasksGainingAnyObligation": 4 } }, "pairedExploratoryEffects": { "coverage-minus-dense": { "ndcg5": { "n": 12, "meanDifference": -0.1083618559067266, "percentile95": [ -0.2307385512538276, 0.022946095040286302 ], "wins": 4, "losses": 8, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "agreedObligationCoverage5": { "n": 12, "meanDifference": 0.125, "percentile95": [ -0.041666666666666664, 0.3333333333333333 ], "wins": 4, "losses": 2, "ties": 6, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "labelRedundancy": { "n": 12, "meanDifference": -0.2127187627187627, "percentile95": [ -0.34812271062271066, -0.07117775742775753 ], "wins": 1, "losses": 10, "ties": 1, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "embeddingRedundancy": { "n": 12, "meanDifference": -0.23078855679271149, "percentile95": [ -0.2799310444947847, -0.18001999607794641 ], "wins": 0, "losses": 12, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." } }, "coverage-minus-typed": { "ndcg5": { "n": 12, "meanDifference": -0.10551516962129533, "percentile95": [ -0.22790090537035515, 0.02447006655901562 ], "wins": 4, "losses": 8, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "agreedObligationCoverage5": { "n": 12, "meanDifference": 0.125, "percentile95": [ -0.041666666666666664, 0.3333333333333333 ], "wins": 4, "losses": 2, "ties": 6, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "labelRedundancy": { "n": 12, "meanDifference": -0.21867114367114368, "percentile95": [ -0.35392361111111115, -0.07481799450549458 ], "wins": 1, "losses": 10, "ties": 1, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "embeddingRedundancy": { "n": 12, "meanDifference": -0.22633436256592296, "percentile95": [ -0.27339113154532957, -0.17750198363268466 ], "wins": 0, "losses": 12, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." } }, "typed-minus-dense": { "ndcg5": { "n": 12, "meanDifference": -0.002846686285431277, "percentile95": [ -0.00854005885629383, 0.0 ], "wins": 0, "losses": 1, "ties": 11, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "agreedObligationCoverage5": { "n": 12, "meanDifference": 0.0, "percentile95": [ 0.0, 0.0 ], "wins": 0, "losses": 0, "ties": 12, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "labelRedundancy": { "n": 12, "meanDifference": 0.0059523809523809494, "percentile95": [ 0.0, 0.01785714285714285 ], "wins": 1, "losses": 0, "ties": 11, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "embeddingRedundancy": { "n": 12, "meanDifference": -0.004454194226788545, "percentile95": [ -0.013362582680365637, 0.0 ], "wins": 0, "losses": 1, "ties": 11, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." } }, "oracle-minus-dense": { "ndcg5": { "n": 12, "meanDifference": 0.251576230016991, "percentile95": [ 0.1340968178421087, 0.3942994424270865 ], "wins": 11, "losses": 0, "ties": 1, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "agreedObligationCoverage5": { "n": 12, "meanDifference": 0.16666666666666666, "percentile95": [ 0.020833333333333332, 0.3541666666666667 ], "wins": 4, "losses": 0, "ties": 8, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "labelRedundancy": { "n": 12, "meanDifference": 0.1165651478151478, "percentile95": [ -0.00586725196100196, 0.2659278395215895 ], "wins": 6, "losses": 3, "ties": 3, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." }, "embeddingRedundancy": { "n": 12, "meanDifference": -0.057801041485365336, "percentile95": [ -0.10680824212538052, -0.010670594594377053 ], "wins": 3, "losses": 9, "ties": 0, "resamples": 10000, "seed": 129300, "interpretation": "Exploratory descriptive paired-task interval; no confirmatory test." } } }, "poolMissingObligations": 4, "oracleMissesDueToBudget": 0 } }, "tasks": [ { "task": "A01", "split": "development", "origin": "public-human-question", "domain": "mathematics", "question": "For a sequence of bounded real or complex functions on a set X, how do pointwise and uniform convergence differ? Explain the role of dependence on x in the choice of N, relate the supremum formulation to the pointwise condition, and show which implication holds and why the converse can fail.", "requirements": [ "Explain the quantifier order and whether N may depend on x.", "Connect the supremum error bound with simultaneous control over the domain.", "Justify the valid implication between the convergence notions.", "Give a concrete example distinguishing the converse." ], "poolSize": 13, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 1471, 1470, 3375, 4524, 4559 ], "ndcg5": 0.9058621177053621, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 2 ], "agreedAssignments": 6, "labelRedundancy": 0.5, "embeddingRedundancy": 0.36565157396523496, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 1471, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 1, 2 ] }, { "id": 1470, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [] }, { "id": 3375, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4524, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4559, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 1471, 1470, 3375, 4524, 4559 ], "ndcg5": 0.9058621177053621, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 2 ], "agreedAssignments": 6, "labelRedundancy": 0.5, "embeddingRedundancy": 0.36565157396523496, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 1471, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 1, 2 ] }, { "id": 1470, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [] }, { "id": 3375, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4524, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4559, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 1470, 1988, 4835, 4524, 3376 ], "ndcg5": 0.690525666709447, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.2, "embeddingRedundancy": 0.10567987565003892, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 1470, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 1, 2 ] }, { "id": 1988, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 4835, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4524, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3376, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 3 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 }, "oracle": { "top5": [ 1470, 1471, 1988, 4521, 4522 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.5, "embeddingRedundancy": 0.25511886247022886, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 1470, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 1, 2 ] }, { "id": 1471, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [] }, { "id": 1988, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 4521, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4522, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 3 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 1287, "coverageMaximizingSubsets": 285, "coveredObligations": 4 }, "poolItems": [ { "id": 1470, "name": "pointwise convergence", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2 ], "definition": "For each fixed input and every positive error tolerance, all terms beyond a sufficiently late stage approximate the limiting value within that tolerance. The stage may depend on the input; this does not guarantee one stage that works throughout the domain." }, { "id": 1471, "name": "uniform convergence", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2 ], "definition": "For every positive error tolerance, all terms beyond one sufficiently late stage approximate a limiting function within that tolerance at every input in the declared domain. The stage may depend on the tolerance but not on the input." }, { "id": 1988, "name": "the converse of a conditional statement", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "A conditional statement formed by exchanging the original hypothesis and conclusion. The converse requires its own justification; the truth of an implication alone does not establish the implication in reverse." }, { "id": 3373, "name": "Taylor series", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "A power series formed from a function's derivatives at a point. A function may possess derivatives of every order without equaling its Taylor series, so smoothness alone does not justify the representation." }, { "id": 3375, "name": "analytic function", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "A function locally equal to a convergent power series. Analyticity is stronger than arbitrary smoothness for real functions; it permits conclusions that should not be transferred to every differentiable function." }, { "id": 3376, "name": "complex analysis", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "The study of functions of complex variables, especially complex-differentiable functions. Complex differentiability imposes strong local structure unlike unrestricted dependence on two real variables." }, { "id": 4521, "name": "dominated convergence", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "For measurable functions converging almost everywhere and bounded in absolute value by one integrable function, the limit may pass through the integral; the functions also converge in the integral of absolute difference." }, { "id": 4522, "name": "monotone convergence of integrals", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "For a nondecreasing sequence of nonnegative measurable functions, the integral of the pointwise limit equals the limit of their integrals, allowing infinity." }, { "id": 4524, "name": "modes of stochastic convergence", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Different convergence requirements on random-variable sequences, including convergence in distribution, in probability and almost surely. A claim must state its mode because these preserve different information and have different implication strengths." }, { "id": 4559, "name": "convergence in probability", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A random-variable sequence converges in probability to a target if, for every positive tolerance, the probability that their distance exceeds that tolerance tends to zero." }, { "id": 4560, "name": "almost sure convergence", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Convergence of a random-variable sequence to its target on an event of probability one. Realizations outside that event may fail to converge even though the exceptional event has probability zero." }, { "id": 4714, "name": "convex proximal operator", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "For an appropriate proper closed convex penalty, mapping a point to the minimizer of that penalty plus a squared-distance term centered at the point. It generalizes projection and can encode nonsmooth penalties." }, { "id": 4835, "name": "Lamport logical clock", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A per-process counter updated so that an event preceding another by the happened-before relation receives a smaller timestamp. The converse implication does not generally hold." } ] }, { "task": "A02", "split": "final", "origin": "public-human-question", "domain": "mathematics", "question": "Why are Sobolev spaces a natural setting for PDEs, and how do weak derivatives, completeness, and embedding results play different roles? For the Laplacian on a bounded domain with smooth boundary, explain what would justify compactness of a Green operator and existence or smoothness of a Rayleigh-quotient minimizer, distinguishing what follows from the function space from what needs additional PDE assumptions.", "requirements": [ "Explain the motivation for weak derivatives and weak solutions relative to classical differentiability.", "Distinguish completeness from the compactness needed for an existence argument.", "Identify the roles and required hypotheses of relevant embedding or compactness results.", "Separate existence of a minimizer from regularity or smoothness, identifying additional assumptions." ], "poolSize": 14, "poolAttainableObligations": [ 1, 2 ], "poolMissingObligations": [ 0, 3 ], "metrics": { "dense": { "top5": [ 3381, 4512, 4348, 4513, 3557 ], "ndcg5": 0.6082561650962087, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 1, 2 ], "agreedAssignments": 5, "labelRedundancy": 0.6, "embeddingRedundancy": 0.20223519622341657, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 3381, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 4512, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 4348, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 4513, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] }, { "id": 3557, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3381, 4512, 4348, 4513, 3557 ], "ndcg5": 0.6082561650962087, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 1, 2 ], "agreedAssignments": 5, "labelRedundancy": 0.6, "embeddingRedundancy": 0.20223519622341657, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 3381, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 4512, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 4348, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 4513, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] }, { "id": 3557, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 4512, 3343, 3373, 3381, 3306 ], "ndcg5": 0.5008949726942703, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 1, 2 ], "agreedAssignments": 4, "labelRedundancy": 0.5, "embeddingRedundancy": 0.08232611241888285, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 4512, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 3343, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 3373, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3381, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 3306, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 4485, 3343, 4348, 4513, 3381 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 1, 2 ], "agreedAssignments": 8, "labelRedundancy": 0.75, "embeddingRedundancy": 0.22466563641090537, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 4485, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 1, 2 ] }, { "id": 3343, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] }, { "id": 4348, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 4513, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] }, { "id": 3381, "agreedObligations": [ 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 2002, "coverageMaximizingSubsets": 1715, "coveredObligations": 2 }, "poolItems": [ { "id": 3306, "name": "rational function", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A quotient of polynomials with a denominator that is not the zero polynomial. At points where that denominator vanishes, the displayed expression needs separate domain analysis even if algebraic cancellation gives a related simpler expression." }, { "id": 3343, "name": "compactness", "grade": 2.5, "judgeA": 3, "judgeB": 2, "agreedObligations": [ 1, 2 ], "definition": "A topological property under which every open cover has a finite subcover. Closed and bounded subsets of Euclidean space are compact, but that characterization does not apply unchanged to every space." }, { "id": 3346, "name": "manifold", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A space that locally resembles Euclidean space of a specified dimension, with additional assumptions such as smoothness when required. Local Euclidean resemblance does not fix the space's global topology or curvature." }, { "id": 3352, "name": "differential geometry", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The study of geometric structures through calculus, especially on smooth manifolds. It relates local differential quantities to global geometry while retaining the assumptions that make derivatives and smooth structures available." }, { "id": 3373, "name": "Taylor series", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A power series formed from a function's derivatives at a point. A function may possess derivatives of every order without equaling its Taylor series, so smoothness alone does not justify the representation." }, { "id": 3381, "name": "Green's function", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "The response kernel associated with a linear differential operator and specified boundary conditions, used to construct solutions from source terms. Changing the operator or boundary problem changes the kernel." }, { "id": 3557, "name": "Maxwell's equations", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "The classical field equations relating electric and magnetic fields to charge, current and one another. Applying them to matter requires constitutive information and boundary conditions; they do not alone specify every material's response." }, { "id": 3687, "name": "radiant flux", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Radiant energy transferred per unit time across a defined description of a source or surface. It measures physical power without weighting wavelengths by human visual sensitivity." }, { "id": 4348, "name": "Hilbert space", "grade": 2.5, "judgeA": 3, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "A real or complex inner-product space complete in the norm induced by that inner product. It supports geometric notions such as length, angle and orthogonal projection in finite or infinite dimensions; a physical state interpretation requires a separate model." }, { "id": 4483, "name": "bounded linear operator", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "A linear map between normed spaces whose output norm is bounded by one finite constant times the input norm. This is equivalent to continuity for linear maps between normed spaces; linearity alone is insufficient in infinite dimensions." }, { "id": 4485, "name": "compact linear operator", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2 ], "definition": "A linear operator between normed spaces sending bounded sets to sets with compact closure. Equivalently, images of bounded sequences have convergent subsequences; boundedness alone need not imply this in infinite dimensions." }, { "id": 4512, "name": "weak topology of a vector space", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "The coarsest topology on a topological vector space that keeps all its originally continuous linear functionals continuous. It can distinguish convergence from convergence in a stronger norm topology." }, { "id": 4513, "name": "weak convergence in a Hilbert space", "grade": 2.5, "judgeA": 3, "judgeB": 2, "agreedObligations": [ 1, 2 ], "definition": "Convergence in the weak topology of a Hilbert space. Norm convergence implies weak convergence, but an infinite orthonormal sequence can converge weakly to zero without its norms tending to zero." }, { "id": 4747, "name": "graph Laplacian", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A matrix representation of a graph’s discrete Laplace operator, conventionally using the negative-operator sign. It connects graph structure with spectra, diffusion and spectral embeddings; normalized and unnormalized variants differ." } ] }, { "task": "A03", "split": "final", "origin": "public-human-question", "domain": "mathematics", "question": "For a general topological space, is there a homology theory whose zeroth group is generated by connected components rather than path components or quasicomponents? Explain the distinctions among these three kinds of components, identify the relevant behavior of singular and Cech homology, and state what assumptions or relaxed axioms matter for a proposed theory.", "requirements": [ "Distinguish connected components, path components, and quasicomponents.", "Explain why the stated zeroth-homology counting goals can differ.", "Assess the comparison with singular and Cech homology under clearly stated hypotheses.", "Identify an appropriate theory or explain the obstruction and any axioms being relaxed." ], "poolSize": 13, "poolAttainableObligations": [ 0, 1 ], "poolMissingObligations": [ 2, 3 ], "metrics": { "dense": { "top5": [ 4466, 4465, 3344, 4468, 4467 ], "ndcg5": 0.7155302697944608, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 1 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.34567069594282185, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 4466, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4465, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3344, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 4468, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4467, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 4466, 4465, 3344, 4468, 4467 ], "ndcg5": 0.7155302697944608, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 1 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.34567069594282185, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 4466, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4465, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3344, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 4468, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4467, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 4466, 4467, 3344, 4974, 3309 ], "ndcg5": 0.6700577470584822, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 1 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.11764039862889726, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 4466, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4467, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3344, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 4974, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3309, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3344, 4466, 3309, 3340, 4465 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 1 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.24003263184083679, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 3344, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 4466, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3309, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3340, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4465, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 1287, "coverageMaximizingSubsets": 495, "coveredObligations": 2 }, "poolItems": [ { "id": 438, "name": "James: integrity made visible in conduct", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A reading frame that tests professed faith through action, speech and treatment of vulnerable people, holding the call to undivided loyalty beside the exposed contradiction of praising God while cursing people or withholding earned wages." }, { "id": 2857, "name": "phase space", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The space of possible states of a physical system under a specified set of state variables. A point denotes a state and a path its evolution; in classical mechanics the variables commonly include positions and momenta, not positions alone." }, { "id": 3309, "name": "mathematical group", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A set with an associative binary operation, an identity element and an inverse for every element. The operation need not commute; the axioms describe compositional structure across many different kinds of objects." }, { "id": 3340, "name": "topological space", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A set equipped with a collection of open subsets satisfying the topology axioms. It supports continuity and neighbourhoods without necessarily assigning numerical distances." }, { "id": 3344, "name": "topological connectedness", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "A space is connected when it cannot be divided into two disjoint nonempty open parts. Connectedness need not guarantee that every pair of points can be joined by a continuous path." }, { "id": 4447, "name": "Lie group", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A group that is also a smooth manifold, with smooth multiplication and inversion. It provides a structure for continuous symmetries; merely being a topological group does not by itself state the differentiable structure." }, { "id": 4465, "name": "chain complex", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A sequence of abelian groups or modules and boundary maps such that two consecutive maps compose to zero. Each incoming image lies in the next kernel; homology measures the part of the kernel not supplied by that image." }, { "id": 4466, "name": "homology of a chain complex", "grade": 1.5, "judgeA": 2, "judgeB": 1, "agreedObligations": [], "definition": "The graded collection of quotient groups formed from cycles modulo boundaries: the kernel of a boundary map divided by the image of the preceding map. It records failures of exactness and can yield invariants of spaces through an associated chain complex." }, { "id": 4467, "name": "cohomology", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Algebraic invariants derived from cochain complexes, often related to functions, pullbacks and local-to-global obstructions. Some cohomology theories include a product giving additional structure; the chosen theory and coefficients matter." }, { "id": 4468, "name": "fundamental group", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The group of based loops in a topological space modulo continuous deformation that fixes the basepoint, with loop concatenation as multiplication. It records one-dimensional homotopy information but is not a complete invariant of arbitrary spaces." }, { "id": 4473, "name": "Hausdorff space", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A topological space in which any two distinct points have disjoint neighborhoods. This separation property ensures uniqueness of convergent limits; it does not require a metric or make every sequence converge." }, { "id": 4474, "name": "separable topological space", "grade": 0.5, "judgeA": 0, "judgeB": 1, "agreedObligations": [], "definition": "A topological space containing a countable dense subset, so every nonempty open set meets that subset. The entire space may still be uncountable; separability is not the Hausdorff separation condition." }, { "id": 4974, "name": "actor–observer attribution hypothesis", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The proposed tendency for people to explain their own behavior more through situations and others' behavior more through dispositions. Broad evidence has not consistently supported the original general claim, so it should be treated as a hypothesis sensitive to conditions rather than a universal law." } ] }, { "task": "A04", "split": "final", "origin": "public-human-question", "domain": "mathematics", "question": "What is the intuitive relationship between singular value decomposition and principal component analysis when reducing a high-dimensional data matrix? Explain what the matrix factorization supplies and what PCA adds as a statistical interpretation or objective. Distinguish applying SVD to raw versus centered data, and explain which retained information and directions are being compared.", "requirements": [ "Relate the relevant singular vectors and singular values to principal directions and variance.", "Distinguish a general matrix factorization from PCA's data-analysis objective.", "Explain the effect of centering and state any scaling conventions.", "Clarify what information dimension reduction preserves and what it discards." ], "poolSize": 13, "poolAttainableObligations": [ 0, 1, 3 ], "poolMissingObligations": [ 2 ], "metrics": { "dense": { "top5": [ 3333, 4638, 4495, 4640, 4492 ], "ndcg5": 0.974679678148328, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.5714285714285714, "embeddingRedundancy": 0.4001360282323877, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3333, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 4638, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [] }, { "id": 4495, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 4640, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4492, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3333, 4638, 4495, 4640, 4492 ], "ndcg5": 0.974679678148328, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.5714285714285714, "embeddingRedundancy": 0.4001360282323877, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3333, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 4638, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [] }, { "id": 4495, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 4640, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4492, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 4638, 4495, 4852, 4154, 4492 ], "ndcg5": 0.6585593239947757, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.15760122923806122, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 4638, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 4495, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 4852, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4154, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4492, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3333, 4638, 4495, 3325, 4492 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.5714285714285714, "embeddingRedundancy": 0.3998633518028883, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3333, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 4638, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [] }, { "id": 4495, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 3325, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4492, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 1287, "coverageMaximizingSubsets": 825, "coveredObligations": 3 }, "poolItems": [ { "id": 3325, "name": "matrix", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "A rectangular array of elements arranged in rows and columns, with operations defined for the intended algebra. A matrix can represent a linear map relative to chosen bases, but the array alone does not specify a physical interpretation." }, { "id": 3333, "name": "singular value decomposition", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 3 ], "definition": "A factorization of a real or complex matrix into orthogonal or unitary factors and a diagonal array of nonnegative singular values. It separates input directions, scaling and output directions, supporting rank and approximation analysis." }, { "id": 4154, "name": "property graph", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A graph data model in which identifiable nodes and directed relationships can carry properties, commonly represented as key-value pairs. Relationships may have their own attributes; an attached property supplies recorded information rather than an automatic proof of the relationship it qualifies." }, { "id": 4492, "name": "positive-semidefinite matrix", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A real symmetric or complex Hermitian matrix whose quadratic form is nonnegative for every vector. Nonzero vectors may have zero value, so singularity is compatible with semidefiniteness." }, { "id": 4495, "name": "matrix factorization", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "A representation of a matrix as a product of matrices chosen to expose structure or support computation. Different factorizations serve different tasks and can have different existence, uniqueness and numerical-stability conditions." }, { "id": 4496, "name": "QR factorization", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A matrix factorization A = QR, with Q orthogonal in the real square case or unitary in the complex square case, and R upper triangular. It supports least-squares calculations and eigenvalue methods." }, { "id": 4497, "name": "Cholesky factorization", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A factorization of a Hermitian positive-definite matrix as a lower triangular matrix times its conjugate transpose. It exploits positive definiteness and is not a general factorization for every square matrix." }, { "id": 4502, "name": "sparse matrix", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A matrix with sufficiently many zero entries for the nonzero pattern to matter to storage or computation. There is no universal numerical cutoff separating sparse from dense matrices." }, { "id": 4638, "name": "principal component analysis", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 3 ], "definition": "A linear coordinate transformation identifying mutually orthogonal directions of descending data variance, often retaining a subset for dimensionality reduction." }, { "id": 4640, "name": "independent component analysis", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Separating multivariate observations into additive components by exploiting an assumed independent source structure, typically with non-Gaussian components. It goes beyond merely removing pairwise correlation." }, { "id": 4642, "name": "nonnegative matrix factorization", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Representing or approximating a nonnegative matrix as a product of nonnegative factors. The constraints support additive component descriptions without requiring subtraction." }, { "id": 4653, "name": "linear discriminant analysis", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Finding linear combinations of features that distinguish labeled groups, for classification or supervised dimension reduction. Statistical variants impose distribution and covariance assumptions on class-conditional data." }, { "id": 4852, "name": "ABA synchronization problem", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A location changes from A to another value and back to A, causing a later equality check to miss intervening changes that matter to correctness." } ] }, { "task": "A05", "split": "final", "origin": "public-human-question", "domain": "physics", "question": "For a black body radiating into empty space, how should the entropy carried by radiation be reconciled with the body's entropy loss? Compare the expressions U/T and 4U/(3T), distinguish emission into a vacuum from reversible expansion of a photon gas against a piston, and explain how energy balance, work, and reversibility affect the comparison.", "requirements": [ "State which systems and temperatures the entropy expressions describe.", "Distinguish energy loss by the body from radiation energy and any mechanical work.", "Compare the free-emission setup with a reversible photon-gas piston process.", "Explain the entropy balance and whether reversibility can be assumed in each setup." ], "poolSize": 19, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3580, 3579, 4399, 3416, 4768 ], "ndcg5": 0.5304759541593448, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 1 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.3130482736140937, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3580, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 3579, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] }, { "id": 4399, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3416, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4768, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3580, 3579, 4399, 3416, 4768 ], "ndcg5": 0.5304759541593448, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 1 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.3130482736140937, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3580, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 3579, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] }, { "id": 4399, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3416, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4768, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3580, 3416, 6047, 3510, 4768 ], "ndcg5": 0.5212267836909997, "agreedObligationCoverage5": 0.25, "coveredObligations": [ 1 ], "agreedAssignments": 2, "labelRedundancy": 0.5, "embeddingRedundancy": 0.22497403219177858, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3580, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 3416, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 6047, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3510, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 4768, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [ 0 ], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 1, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3510, 3559, 3562, 3578, 4381 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.2, "embeddingRedundancy": 0.3824512755147863, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3510, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 3559, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 3562, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] }, { "id": 3578, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 4381, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 2, 3 ], "lostCountVsDense": 0, "gainedCountVsDense": 2 } }, "oracleEnumeration": { "subsetsEnumerated": 11628, "coverageMaximizingSubsets": 104, "coveredObligations": 4 }, "poolItems": [ { "id": 983, "name": "documented and reversible conservation work", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A draft heritage practice requiring justified interventions, public works records and reversibility where feasible, with emergency readiness and later damage assessment. Conserving a site is distinguished from using works to establish unilateral territorial or custodial changes." }, { "id": 2604, "name": "interstellar medium", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The gas, dust, particles and radiation between stellar systems within a galaxy. Its phases differ in temperature, density and ionization, and interact through processes such as turbulence, magnetic fields, cooling and stellar feedback; interstellar space is not simply empty." }, { "id": 2619, "name": "gravitational wave", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A propagating disturbance of spacetime curvature that carries energy away from changing gravitational sources. In general relativity it travels at the vacuum speed of light and can produce measurable tidal distortions; it is distinct from electromagnetic radiation and static gravitational attraction." }, { "id": 3058, "name": "greenhouse effect", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Warming associated with atmospheric absorption and emission of infrared radiation that changes a planet's radiative energy balance. The effect alters heat loss rather than stopping it completely, and its strength depends on atmospheric and surface conditions." }, { "id": 3416, "name": "Shannon entropy", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "The expected information content of outcomes drawn from a specified distribution, measured in units set by the logarithm's base. It concerns uncertainty in that distribution, not automatically semantic importance or thermodynamic entropy." }, { "id": 3510, "name": "energy", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "A conserved physical quantity transferred or transformed through processes such as work, heating and radiation. Its forms depend on the system and model; possessing energy does not mean all of it can be converted into useful work." }, { "id": 3559, "name": "electromagnetic radiation", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "Energy carried by propagating electromagnetic fields, spanning a wide range of frequencies. Its interaction with matter depends on frequency and material properties; visible light is only one part of the spectrum." }, { "id": 3562, "name": "temperature", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "An intensive thermodynamic quantity characterizing thermal state and the direction of spontaneous heat transfer between systems. It is not the total heat or energy contained in a body." }, { "id": 3572, "name": "Gibbs free energy", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A thermodynamic potential formed from enthalpy minus temperature times entropy. At fixed temperature and pressure under appropriate constraints, its change characterizes equilibrium and available non-expansion work." }, { "id": 3578, "name": "thermal radiation", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "Electromagnetic radiation emitted because of matter's thermal state. Its spectrum and power depend on temperature and emissive properties; radiation does not require a material medium between emitter and receiver." }, { "id": 3579, "name": "black-body radiation", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "The equilibrium electromagnetic emission of an ideal perfect absorber, with a spectrum determined by temperature. Real surfaces may depart from this ideal through wavelength- and direction-dependent emissivity." }, { "id": 3580, "name": "Stefan–Boltzmann law", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "The fourth-power temperature relation for total radiant exitance of an ideal black body. Net radiative exchange also depends on surroundings, geometry and emissivity rather than emitted power alone." }, { "id": 3612, "name": "quantum vacuum state", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "The lowest-energy state of a quantum field theory under specified conditions. It is not an ordinary empty container lacking fields, and its properties depend on the theory and background." }, { "id": 4381, "name": "radiative equilibrium", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "A condition of balanced radiative energy gain and loss for the specified object or region. This balance concerns radiation; other transfers or internal processes can prevent full thermal or thermodynamic equilibrium." }, { "id": 4397, "name": "radiant exitance", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "Radiant power emitted from a surface per unit surface area, measured in watts per square metre. It is distinct from irradiance received by a surface and from radiant intensity per solid angle; equal units alone do not make different flux quantities identical." }, { "id": 4399, "name": "thermal emissivity", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The ratio of a surface’s thermal emission to that of an ideal blackbody at the same temperature under corresponding spectral and directional conditions. Composition and surface structure matter; a single grey-body value is a modelling approximation." }, { "id": 4768, "name": "Rényi entropy", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "For a discrete probability distribution, the order-α entropy is the logarithm of the sum of α-powers of probabilities divided by 1−α, with limiting definitions at special orders. The order changes sensitivity to concentrated outcomes." }, { "id": 5728, "name": "aseptic practice", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Practices intended to prevent introduction or transfer of disease-causing microorganisms in a procedure or protected field, using controls that may include sterile materials." }, { "id": 6047, "name": "stimulated emission", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "A transition in which an electromagnetic field stimulates an excited system to release energy into the corresponding radiation mode. It supports optical amplification when emission overcomes competing absorption and losses." } ] }, { "task": "A06", "split": "final", "origin": "public-human-question", "domain": "physics", "question": "How can a minimum entropy-production principle near equilibrium be compared with claims that nonequilibrium systems maximize entropy production? Identify whether the principles optimize the same quantity under the same constraints, distinguish a steady-state selection rule from time evolution toward that state, and explain which regimes and assumptions determine whether there is a real contradiction.", "requirements": [ "Specify the quantities and constraints that each claimed extremum principle uses.", "Distinguish an extremum over admissible states from a statement about time derivatives.", "State the role of the near-equilibrium or linear regime.", "Assess the apparent contradiction without treating a general maximum-production claim as established." ], "poolSize": 13, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3568, 3565, 4376, 3569, 3635 ], "ndcg5": 0.49301520849026176, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.2, "embeddingRedundancy": 0.4839198977395568, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3568, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 3565, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 4376, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 3569, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3635, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3568, 3565, 4376, 3569, 3635 ], "ndcg5": 0.49301520849026176, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.2, "embeddingRedundancy": 0.4839198977395568, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3568, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 3565, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 4376, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 3569, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3635, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3568, 2836, 3588, 2852, 4565 ], "ndcg5": 0.5652961308067935, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.18223225202785714, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3568, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 2836, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 3588, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2852, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 4565, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [ 2 ], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 1, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 2836, 4373, 2852, 3565, 3568 ], "ndcg5": 0.9634824023755321, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.36361917520995696, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 2836, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 4373, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 2852, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 3565, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 3568, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 1287, "coverageMaximizingSubsets": 181, "coveredObligations": 4 }, "poolItems": [ { "id": 2815, "name": "state-space representation", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "A dynamical model expressed through state variables and equations for their evolution and observed outputs. Linear time-invariant systems admit a compact matrix form; state-space modelling also accommodates nonlinear systems with appropriate equations." }, { "id": 2836, "name": "equilibrium point", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "A state unchanged by a dynamical system's evolution rule: a constant solution of a differential equation or a fixed point of a discrete update. Existence of such a state does not establish that nearby trajectories remain near it or converge to it." }, { "id": 2852, "name": "dynamical system", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "A state space together with a rule describing how states evolve with time. The rule may be continuous or discrete and deterministic or stochastic; a trajectory is a particular evolution from stated conditions, not the whole space of possibilities." }, { "id": 3565, "name": "thermodynamic equilibrium", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 2 ], "definition": "A state with no macroscopic tendency toward spontaneous internal change under the imposed constraints, encompassing the relevant thermal, mechanical and chemical balances. Apparent steadiness maintained by external flows is not necessarily equilibrium." }, { "id": 3566, "name": "zeroth law of thermodynamics", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The transitivity of thermal equilibrium: systems each in thermal equilibrium with a third are in thermal equilibrium with one another. It supports the use of temperature as a consistent comparison quantity." }, { "id": 3568, "name": "second law of thermodynamics", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 3 ], "definition": "A thermodynamic constraint expressed by nondecrease of total entropy for an isolated system and by limits on heat-to-work conversion. A subsystem's entropy may decrease when compensating transfers or production occur elsewhere." }, { "id": 3569, "name": "third law of thermodynamics", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "In its perfect-crystal formulation, entropy tends to zero as temperature approaches absolute zero when the ground state is unique. Degeneracy, residual disorder and nonequilibrium states require separate care; the law does not promise an achievable zero-temperature process." }, { "id": 3570, "name": "thermodynamic entropy", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "A thermodynamic state quantity whose changes constrain heat transfer and irreversible processes, related statistically to accessible microstates. It is not simply a visual measure of messiness, nor automatically the same quantity as semantic uncertainty." }, { "id": 3588, "name": "equipartition theorem", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A classical equilibrium result assigning an average energy contribution to each quadratic degree of freedom. Its assumptions can fail for quantum or nonequilibrium systems, so it is not a universal rule for every mode." }, { "id": 3635, "name": "chemical equilibrium", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A state in which forward and reverse reaction rates balance so macroscopic composition remains constant under the conditions. Reactions continue microscopically; equilibrium is not generally equal quantities of reactants and products." }, { "id": 4373, "name": "steady state", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "A condition in which the selected state variables do not change with time under a specified description. Continuing inputs and outputs may maintain it; constancy of those variables does not imply equilibrium, absence of activity or stability after a disturbance." }, { "id": 4376, "name": "thermodynamics", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "The study of heat, work, temperature, energy and entropy through macroscopic states, transformations and constraints. It can constrain which processes are possible without by itself specifying their rate or complete microscopic mechanism." }, { "id": 4565, "name": "finite-population correction", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Adjustment of sampling uncertainty for sampling without replacement from a fixed finite population. Sampling a larger fraction reduces remaining sampling uncertainty relative to an otherwise comparable with-replacement calculation." } ] }, { "task": "A07", "split": "final", "origin": "public-human-question", "domain": "physics", "question": "When light propagates through a transparent material, how does the microscopic response produce a refractive phase velocity? Compare a repeated absorption-and-reemission story with a coherent wave-response explanation, distinguish phase, group, and signal velocities, and explain how a continuous dispersion relation can arise despite discrete atomic transitions. State which material and frequency assumptions your explanation requires.", "requirements": [ "Compare incoherent absorption and reemission with coherent polarization or scattering as explanations.", "Connect microscopic response to macroscopic phase delay or refractive index.", "Distinguish phase, group, and signal velocities and their limits of interpretation.", "Explain the relationship between resonances, transparency, and frequency-dependent dispersion." ], "poolSize": 12, "poolAttainableObligations": [ 1, 2, 3 ], "poolMissingObligations": [ 0 ], "metrics": { "dense": { "top5": [ 3548, 3686, 3680, 6065, 2340 ], "ndcg5": 0.6533463439927868, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.34007996802025714, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3548, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3686, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 3680, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 6065, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2340, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3548, 3686, 3680, 6065, 2340 ], "ndcg5": 0.6533463439927868, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.34007996802025714, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3548, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3686, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 3680, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 6065, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2340, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3548, 3686, 3680, 4887, 4311 ], "ndcg5": 0.6011251993361093, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.16776628382925263, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3548, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3686, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 3680, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 4887, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4311, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3561, 3680, 3681, 3548, 3686 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.5, "embeddingRedundancy": 0.466042553026083, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3561, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 3680, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 3681, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 3548, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3686, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 792, "coverageMaximizingSubsets": 260, "coveredObligations": 3 }, "poolItems": [ { "id": 2340, "name": "an opaque aggregate made from transmitting constituents", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "A material can lose transparency when divided into many separated pieces or layers because the intervening boundaries repeatedly redirect light. Reuniting the pieces or changing what occupies the gaps can restore transmission without requiring a different constituent substance." }, { "id": 3548, "name": "wave dispersion", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "Dependence of propagation speed on frequency, causing different spectral components to accumulate different phases. Pulse spreading can result; dispersion is distinct from simple attenuation of every component." }, { "id": 3561, "name": "vacuum speed of light", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2 ], "definition": "The invariant speed c of light in vacuum, a defining constant of relativity and SI units. Propagation in material media can be slower; phase, group and signal velocities must not be treated as interchangeable in every setting." }, { "id": 3675, "name": "frequency", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "The number of repetitions of a periodic event per unit time, or a specified rate of occurrence. The counted event and time reference must be stated; frequency is not the same as the speed at which a pattern travels through space." }, { "id": 3680, "name": "phase velocity", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2 ], "definition": "The speed at which a point of constant phase of a wave travels. It can differ from group or signal velocity in a dispersive medium and does not alone specify the speed of transmitted information." }, { "id": 3681, "name": "group velocity", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2 ], "definition": "The propagation speed associated with a wave packet's envelope, commonly derived from the change of angular frequency with wave number. Its relation to energy or information transport requires conditions, especially in strongly dispersive media." }, { "id": 3686, "name": "refractive index", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "The ratio of vacuum light speed to phase speed in a medium at a specified frequency under the relevant optical description. It may depend on direction and frequency; it is not simply a measure of opacity." }, { "id": 4311, "name": "principle of interpretive charity", "grade": 0.5, "judgeA": 0, "judgeB": 1, "agreedObligations": [], "definition": "A methodological preference for a coherent and rational interpretation of another’s words when the evidence permits it. Charity avoids attributing needless error; it does not authorize rewriting a position into one the speaker did not express or declaring it true." }, { "id": 4887, "name": "Petri net", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A discrete-event model with places, tokens and transitions. Enabled transitions consume specified input tokens and produce output tokens, representing state change and concurrency." }, { "id": 5950, "name": "mechanical impedance", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The frequency-dependent relation between applied harmonic force and resulting velocity at a stated mechanical point or port. It includes both magnitude and phase, and is distinct from static stiffness." }, { "id": 5951, "name": "acoustic impedance", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The frequency-domain relation between sound pressure and acoustic volume flow through a stated port. It differs from specific acoustic impedance, which relates pressure to particle velocity and has different units." }, { "id": 6065, "name": "Brillouin scattering", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Scattering of light by material fluctuations or waves that change refractive properties and exchange energy and momentum with the radiation. The resulting frequency shift can reveal collective mechanical behavior." } ] }, { "task": "A08", "split": "development", "origin": "public-human-question", "domain": "physics", "question": "How are inertial mass, gravitational mass, rest mass, invariant mass, and relativistic mass related? Organize the terms by what they measure and by their dependence on reference frame, distinguish experimental equivalence from a change of convention, and explain which terminology is appropriate when comparing a body at rest with the same body in motion.", "requirements": [ "Explain the operational distinction between inertial and gravitational mass.", "Relate rest mass and invariant mass while stating the scope of the terms.", "Explain what relativistic mass denotes and how frame dependence enters.", "Separate empirical equivalence claims from alternative definitions or conventions." ], "poolSize": 14, "poolAttainableObligations": [ 0, 2, 3 ], "poolMissingObligations": [ 1 ], "metrics": { "dense": { "top5": [ 3502, 1906, 1902, 5938, 3504 ], "ndcg5": 0.9352480319859555, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.5, "embeddingRedundancy": 0.4734646654509471, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 3502, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 1906, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 1902, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 5938, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3504, "agreedObligations": [ 0 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3502, 1906, 1902, 3503, 5938 ], "ndcg5": 0.9010877965607802, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.5714285714285714, "embeddingRedundancy": 0.4200143347294846, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 3502, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 1906, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 1902, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 3503, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 5938, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3502, 1906, 1075, 2578, 3177 ], "ndcg5": 0.7048304809097502, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.1254896086942512, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3502, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 1906, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 1075, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2578, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3177, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 1906, 3502, 3504, 1902, 3503 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.625, "embeddingRedundancy": 0.4229309220468912, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 1906, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 3502, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 3504, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 1902, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 3503, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 2002, "coverageMaximizingSubsets": 1079, "coveredObligations": 3 }, "poolItems": [ { "id": 1075, "name": "Black Sea commerce as a proposed shared peace interest", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Memo 7 proposes secure commercial navigation, restored grain and commodity movement, and possible joint Black Sea infrastructure or economic cooperation. The intended link is that reliable trade and civilian livelihoods give several states an ongoing interest in maintaining peace." }, { "id": 1902, "name": "invariance of a law distinguished from sameness of measured motion", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "In Einstein’s restricted principle, uniformly translating nonrotating reference systems describe natural processes with the same general laws, even though a body’s measured velocity or direction can differ between them." }, { "id": 1906, "name": "inertial and gravitational mass as distinct roles with an empirical link", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "Einstein distinguishes a body’s resistance to acceleration from its gravitational coupling, then connects them through the observed common acceleration of freely falling bodies under the stated conditions. Equality is physically informative rather than merely a shared use of the word mass." }, { "id": 2569, "name": "orbital elements", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Parameters describing an orbit under a chosen convention, reference frame and epoch, commonly using a Keplerian two-body model. Real perturbations make fitted elements change over time, so an element set is not a timeless description of a body's motion." }, { "id": 2578, "name": "axial tilt", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The angle between a body's rotation axis and the normal to its orbital plane, under a stated pole convention. It relates spin orientation to orbital geometry and differs from orbital inclination, which compares an orbit with a reference plane." }, { "id": 3177, "name": "specific impulse", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A propulsion measure relating delivered impulse to propellant consumed. It can be expressed as effective exhaust velocity or, after division by standard gravitational acceleration, in seconds; the convention must be stated." }, { "id": 3502, "name": "mass", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 2 ], "definition": "A physical property associated with inertia and gravitational interaction. In relativity, invariant mass is distinct from frame-dependent energy; specifying a system and its energy content matters more than treating mass simply as a count of material parts." }, { "id": 3503, "name": "weight", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 3 ], "definition": "The force associated with gravity on a body, or in some usage the support force indicated by a scale. The convention must be stated; weight depends on circumstances and is not identical to mass." }, { "id": 3504, "name": "inertia", "grade": 2.5, "judgeA": 3, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "The resistance of a body's motion to change, expressed in classical mechanics by persistence of uniform straight-line motion when net external force vanishes. Translational and rotational changes depend on mass and mass distribution respectively." }, { "id": 3505, "name": "linear momentum", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "The vector quantity describing translational motion, equal to mass times velocity in classical mechanics and defined differently in relativistic settings. Its value depends on reference frame; conservation concerns the specified system and external interactions." }, { "id": 3512, "name": "kinetic energy", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "Energy associated with motion relative to a reference frame. Its classical translational form is one-half mass times speed squared; relativistic motion requires a different expression." }, { "id": 3518, "name": "Newtonian motion laws", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "The classical relations describing inertial motion, momentum change under net force, and paired interaction forces. Their ordinary form assumes an inertial frame and an appropriate classical regime; it is not an unrestricted relativistic or quantum law." }, { "id": 4335, "name": "classical mass conservation", "grade": 1.5, "judgeA": 2, "judgeB": 1, "agreedObligations": [], "definition": "The approximation that total mass is conserved in ordinary chemical and low-energy material processes, with transfers across the system boundary explicitly counted. Relativistic mass–energy exchange limits treating the sum of material masses as universally invariant." }, { "id": 5938, "name": "mass moment of inertia", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A measure of a mass distribution relative to an axis, weighting each mass element by the square of its perpendicular distance. It governs rotational response about that axis; the same object has different values for different axes." } ] }, { "task": "A09", "split": "final", "origin": "public-human-question", "domain": "statistics", "question": "In linear regression, both a confidence interval for the mean response at x0 and a prediction interval for a new response can be centered on the fitted line. What different uncertainty does each interval represent, how do their coverage statements differ, and which interval should be used for estimating an average response versus predicting one new observation?", "requirements": [ "Distinguish the conditional mean response from a future individual response.", "Identify the uncertainty components relevant to each interval.", "Explain the appropriate repeated-sampling coverage interpretation.", "State the model assumptions and match each interval to its intended question." ], "poolSize": 11, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 4692, 4694, 3424, 3494, 4693 ], "ndcg5": 0.9177513973666739, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.6, "embeddingRedundancy": 0.4756106318640027, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 4692, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [ 0, 1, 2, 3 ] }, { "id": 4694, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [] }, { "id": 3424, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] }, { "id": 3494, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] }, { "id": 4693, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 4692, 4694, 3424, 3494, 4693 ], "ndcg5": 0.9177513973666739, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.6, "embeddingRedundancy": 0.4756106318640027, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 4692, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [ 0, 1, 2, 3 ] }, { "id": 4694, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [] }, { "id": 3424, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] }, { "id": 3494, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] }, { "id": 4693, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 4694, 4692, 3494, 4618, 3837 ], "ndcg5": 0.7381204643644838, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.5, "embeddingRedundancy": 0.21006831222522931, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 4694, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 4692, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 3494, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] }, { "id": 4618, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3837, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3424, 4692, 4694, 3437, 3494 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 12, "labelRedundancy": 0.6666666666666666, "embeddingRedundancy": 0.33448528066562233, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3424, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 4692, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 4694, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [] }, { "id": 3437, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 3494, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 462, "coverageMaximizingSubsets": 281, "coveredObligations": 4 }, "poolItems": [ { "id": 1478, "name": "time coverage of an observed average", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The extent to which an average represents variations throughout an interval rather than only the selected moments at which observations were taken." }, { "id": 3424, "name": "confidence interval", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "An interval produced by a procedure with a stated long-run coverage property under its assumptions. A frequentist confidence level describes the procedure's repeated behavior, not generally the posterior probability that a fixed parameter lies in the observed interval." }, { "id": 3437, "name": "linear regression", "grade": 2.5, "judgeA": 3, "judgeB": 2, "agreedObligations": [ 0, 3 ], "definition": "A regression model linear in its coefficients, commonly estimating a conditional mean as a linear combination of predictors. Transformed predictors can appear, and linearity does not itself ensure independent errors or valid causal interpretation." }, { "id": 3494, "name": "uncertainty quantification", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 3 ], "definition": "Characterizing and propagating uncertainty in models, inputs and predictions, including the assumptions used to assign it. A numerical uncertainty range is only as comprehensive as the sources and dependencies represented." }, { "id": 3837, "name": "supererogation", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Morally good action that exceeds what duty requires, so declining to perform it need not be morally wrong. It is distinguished both from an obligation and from an ethically neutral option; what counts as beyond duty depends on the normative account." }, { "id": 4543, "name": "Gaussian-process regression", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Prediction of unobserved function values using a Gaussian-process model and its covariance assumptions. In the geostatistical formulation, suitable assumptions yield best linear unbiased prediction at unsampled locations." }, { "id": 4618, "name": "quantile regression", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Modeling a conditional quantile of a response as a function of predictors, rather than modeling only its conditional mean. Different quantiles can describe different parts of the response distribution." }, { "id": 4623, "name": "regression leverage", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An observation’s potential to affect a fitted regression because of its position in predictor space. In ordinary linear regression its leverage is a diagonal element of the projection or hat matrix." }, { "id": 4692, "name": "prediction interval", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2, 3 ], "definition": "An interval intended to cover a future observation at a stated probability or repeated-sampling rate under a specified predictive model. It includes variability of the future outcome, not only uncertainty about a mean." }, { "id": 4693, "name": "statistical tolerance interval", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A sample-based interval designed, at a stated confidence level, to contain at least a specified fraction of a population. Population content and confidence are separate parameters." }, { "id": 4694, "name": "statistical interval coverage", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 2 ], "definition": "The probability that a specified interval-construction procedure contains its target under repeated sampling: a fixed parameter for confidence intervals or a future outcome for prediction intervals." } ] }, { "task": "A10", "split": "final", "origin": "public-human-question", "domain": "statistics", "question": "Can a nonparametric bootstrap compensate for a small original sample when a parametric inference method seems unreliable? Distinguish avoiding a parametric distribution assumption from acquiring new information, explain what resampling can and cannot approximate, and assess whether thousands of bootstrap replicates may legitimately be treated as thousands of independent observations from the population.", "requirements": [ "Distinguish original sample size from the number of bootstrap replicates.", "Explain the bootstrap approximation and its dependence on the observed sample.", "Discuss the relevance of distributional assumptions and small-sample limitations.", "Assess the validity of treating resampled or compounded outcomes as new independent population data." ], "poolSize": 13, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3442, 2466, 4565, 4667, 4650 ], "ndcg5": 0.9058527658710546, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.42857142857142855, "embeddingRedundancy": 0.3414278759092638, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3442, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [ 0, 1, 2, 3 ] }, { "id": 2466, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [] }, { "id": 4565, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4667, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4650, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3442, 2466, 4565, 4667, 4650 ], "ndcg5": 0.9058527658710546, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.42857142857142855, "embeddingRedundancy": 0.3414278759092638, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3442, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [ 0, 1, 2, 3 ] }, { "id": 2466, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [] }, { "id": 4565, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4667, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4650, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3442, 3432, 4744, 4554, 5752 ], "ndcg5": 0.6294491554097035, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.2, "embeddingRedundancy": 0.1685156888423597, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3442, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [ 0, 1, 2, 3 ] }, { "id": 3432, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 4744, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4554, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5752, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 2466, 3442, 3432, 55, 4650 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.5, "embeddingRedundancy": 0.33613289915719907, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 2466, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [ 0, 2, 3 ] }, { "id": 3442, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 3432, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 55, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4650, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 1287, "coverageMaximizingSubsets": 495, "coveredObligations": 4 }, "poolItems": [ { "id": 55, "name": "inspectable assumption", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "An assumption expressed so another reader can locate it, distinguish it from evidence, challenge it, and observe what changes when it is replaced." }, { "id": 2466, "name": "sampling convergence separated from model adequacy", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 2, 3 ], "definition": "Increasing samples can reduce an estimator’s random error for a specified model while leaving omitted processes, incorrect assumptions, or biased inputs untouched. A progressively smoother result supports convergence within the stated calculation, not completeness or truth of the world it depicts." }, { "id": 3432, "name": "sampling bias", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "A systematic mismatch between a sample's selection process and the population or quantity intended for inference. Increasing sample size need not repair that mismatch." }, { "id": 3442, "name": "bootstrap resampling", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2, 3 ], "definition": "Estimating a statistic's variability by repeatedly resampling from an empirical sample under a specified scheme. The procedure inherits limitations of the sample and resampling model; it does not create missing population information." }, { "id": 4554, "name": "particle filtering", "grade": 0.5, "judgeA": 0, "judgeB": 1, "agreedObligations": [], "definition": "Approximate sequential state inference using weighted samples to represent a posterior distribution under noisy or partial observations. Prediction, weighting and resampling update the representation over time." }, { "id": 4563, "name": "design-based survey inference", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Inference about a target population based on the probabilities specified by a survey sampling design. The design supplies a sampling distribution but does not eliminate measurement, nonresponse or coverage bias." }, { "id": 4565, "name": "finite-population correction", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Adjustment of sampling uncertainty for sampling without replacement from a fixed finite population. Sampling a larger fraction reduces remaining sampling uncertainty relative to an otherwise comparable with-replacement calculation." }, { "id": 4579, "name": "Welch unequal-variance test", "grade": 0.5, "judgeA": 0, "judgeB": 1, "agreedObligations": [], "definition": "An approximate two-sample test of mean equality that estimates variability separately for independent groups rather than assuming equal population variances." }, { "id": 4592, "name": "empirical Bayes estimation", "grade": 0.5, "judgeA": 0, "judgeB": 1, "agreedObligations": [], "definition": "Estimating aspects of a prior distribution from data and then using them for inference. It can approximate a hierarchical Bayesian analysis while treating estimated higher-level parameters as fixed." }, { "id": 4650, "name": "bootstrap aggregation", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "Fitting predictors to bootstrap resamples and combining their outputs by averaging or voting. It is intended to reduce instability arising from the training sample." }, { "id": 4667, "name": "missing not at random", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A missingness mechanism that still depends on unobserved values after conditioning on the relevant observed information. Valid inference then needs further modeling, identifying assumptions or sensitivity analysis." }, { "id": 4744, "name": "treewidth", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The minimum, over a graph’s tree decompositions, of the largest bag size minus one. It measures a structural width relevant to many graph algorithms." }, { "id": 5752, "name": "founder effect", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Genetic change caused when a new population begins from a small, nonrepresentative sample of a source population, potentially changing allele frequencies and reducing variation." } ] }, { "task": "A11", "split": "final", "origin": "public-human-question", "domain": "statistics", "question": "When training data are not independent and identically distributed, which problems concern fitting a model, which concern inference, and which concern estimating generalization? Consider temporally dependent predictors, conditionally independent responses, and more features than observations. Explain what this changes for selecting a LASSO penalty by cross-validation and for using nested validation to report predictive performance.", "requirements": [ "Separate independence from identical-distribution assumptions and state which fail in the example.", "Distinguish computational fitting from inferential and generalization guarantees.", "Assess how dependence affects data splitting and penalty selection.", "Explain what a suitable nested evaluation design would need to respect and what it cannot repair." ], "poolSize": 12, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3498, 4608, 3496, 4636, 4625 ], "ndcg5": 0.7645295418043103, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.7, "embeddingRedundancy": 0.2905327985085588, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3498, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 1, 2, 3 ] }, { "id": 4608, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] }, { "id": 3496, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] }, { "id": 4636, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] }, { "id": 4625, "agreedObligations": [ 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3498, 4608, 3496, 4636, 4625 ], "ndcg5": 0.7645295418043103, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.7, "embeddingRedundancy": 0.2905327985085588, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3498, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 1, 2, 3 ] }, { "id": 4608, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] }, { "id": 3496, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] }, { "id": 4636, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] }, { "id": 4625, "agreedObligations": [ 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3496, 3498, 4538, 5412, 3438 ], "ndcg5": 0.5198567611357467, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.4, "embeddingRedundancy": 0.12902860631471663, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3496, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 3498, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 2 ] }, { "id": 4538, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5412, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3438, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3498, 3499, 3501, 3496, 3437 ], "ndcg5": 0.9657673782048287, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 11, "labelRedundancy": 0.6363636363636364, "embeddingRedundancy": 0.27004866161858937, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3498, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 1, 2, 3 ] }, { "id": 3499, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] }, { "id": 3501, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] }, { "id": 3496, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] }, { "id": 3437, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 0 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 792, "coverageMaximizingSubsets": 265, "coveredObligations": 4 }, "poolItems": [ { "id": 3437, "name": "linear regression", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 1 ], "definition": "A regression model linear in its coefficients, commonly estimating a conditional mean as a linear combination of predictors. Transformed predictors can appear, and linearity does not itself ensure independent errors or valid causal interpretation." }, { "id": 3438, "name": "logistic regression", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "A regression model expressing the log-odds of a binary outcome as a linear combination of predictors. Its coefficients concern odds on that scale, not directly equal changes in outcome probability." }, { "id": 3496, "name": "regularization", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "Adding constraints, penalties or prior structure to stabilize estimation or restrict solutions to an ill-posed or flexible problem. It can improve useful inference while introducing assumptions that must remain visible." }, { "id": 3498, "name": "cross-validation", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2, 3 ], "definition": "Estimating predictive performance by repeatedly fitting on part of a dataset and evaluating on held-out parts under a declared splitting scheme. Leakage, dependence or repeated tuning on evaluation results can compromise the estimate." }, { "id": 3499, "name": "overfitting", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "Fitting idiosyncrasies or noise in training data so closely that performance on relevant new data deteriorates. Excellent training fit can therefore coexist with poor generalization." }, { "id": 3501, "name": "data leakage in predictive modeling", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "Unintended use during model development of information that would not be available at the intended prediction time or independent evaluation. It can make measured performance optimistic without improving real predictive ability." }, { "id": 4538, "name": "hidden Markov model", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A model with an unobserved Markov state process and observations generated conditionally from the current hidden state. Inference uses the observations to estimate hidden states or model parameters." }, { "id": 4599, "name": "widely applicable information criterion", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A Bayesian predictive model criterion based on pointwise posterior predictive fit and a complexity correction. Its asymptotic connection to cross-validation does not make finite-sample predictions certain." }, { "id": 4608, "name": "bias–variance trade-off", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 2 ], "definition": "The tension between systematic prediction error and sensitivity to the training sample when selecting a modeling procedure. Greater flexibility can reduce bias while increasing variance, depending on the model and regime." }, { "id": 4625, "name": "regression multicollinearity", "grade": 2.5, "judgeA": 3, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "Exact or near linear dependence among regression predictors. Exact dependence prevents unique identification of all unrestricted coefficients; near dependence can make their estimates unstable." }, { "id": 4636, "name": "lasso regression", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 2 ], "definition": "Regression with an L1 penalty or equivalent bound on the sum of absolute coefficient values. Its geometry can set coefficients exactly to zero, combining shrinkage with variable selection." }, { "id": 5412, "name": "osmoregulation", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Regulation of water and dissolved-solute balance to keep an organism’s internal fluids within functional conditions. Organisms differ in how closely they match external osmolarity; osmotic balance does not require that internal ionic composition equal the environment." } ] }, { "task": "A12", "split": "development", "origin": "public-human-question", "domain": "statistics", "question": "How much causal interpretation can an observational regression gain by adjusting for many measured covariates? Explain the difference between controlling a statistical association and identifying a causal effect, distinguish useful adjustment from adjustment that introduces bias, and assess how unmeasured variables, model assumptions, and alternatives such as stratification or propensity scores affect confidence in the result.", "requirements": [ "Distinguish a regression coefficient after adjustment from an identified causal effect.", "State the assumptions needed for an adjustment set to support the causal question.", "Explain why measuring more covariates does not automatically improve validity.", "Compare what regression, stratification, or propensity-based methods can and cannot address." ], "poolSize": 14, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3490, 3483, 3435, 3479, 3436 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 13, "labelRedundancy": 0.6923076923076923, "embeddingRedundancy": 0.48801785088800403, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3490, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 1, 2, 3 ] }, { "id": 3483, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0 ] }, { "id": 3435, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [] }, { "id": 3479, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 3436, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3490, 3483, 3435, 3479, 3436 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 13, "labelRedundancy": 0.6923076923076923, "embeddingRedundancy": 0.48801785088800403, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3490, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 1, 2, 3 ] }, { "id": 3483, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0 ] }, { "id": 3435, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [] }, { "id": 3479, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 3436, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3483, 3485, 4569, 4678, 4635 ], "ndcg5": 0.7414701551467148, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.42857142857142855, "embeddingRedundancy": 0.186793533029795, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3483, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 3485, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 4569, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 4678, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4635, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3404, 3435, 3436, 3479, 3483 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 11, "labelRedundancy": 0.6363636363636364, "embeddingRedundancy": 0.4106301731945566, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3404, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] }, { "id": 3435, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [ 1, 2, 3 ] }, { "id": 3436, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 3479, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 3483, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 2002, "coverageMaximizingSubsets": 1465, "coveredObligations": 4 }, "poolItems": [ { "id": 3165, "name": "risk factor", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "A characteristic or exposure associated with a higher probability of an outcome in a specified setting. It may be causal, a marker of other causes or affected by bias; identifying an association does not alone establish a mechanism." }, { "id": 3404, "name": "statistical correlation", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0 ], "definition": "A measure or relation describing association between variables, often quantified by normalized covariance. The chosen coefficient captures particular forms of association and does not establish a causal mechanism." }, { "id": 3435, "name": "confounding", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2, 3 ], "definition": "A mixing of effects that prevents a comparison from isolating the causal relationship of interest, often because other causes influence both exposure and outcome. Adjustment requires a defensible causal structure, not merely adding every available variable." }, { "id": 3436, "name": "regression analysis", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "Statistical modeling of how an outcome's distribution or summary depends on predictor variables. An estimated regression relationship need not be causal, and extrapolation beyond supported conditions requires additional justification." }, { "id": 3472, "name": "experiment", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An inquiry that deliberately controls or changes conditions to examine resulting observations against a question or hypothesis. Its inferential strength depends on design, comparison and measurement, not merely on intervening." }, { "id": 3479, "name": "observational study", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "A study in which investigators observe exposures or conditions without assigning them as experimental interventions. Associations can inform causal inquiry, but causal interpretation requires assumptions about selection, measurement and alternative causes." }, { "id": 3483, "name": "causal inference", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "Drawing conclusions about how changes in one factor would affect another, using evidence together with an explicit causal design or model. Association alone leaves open confounding, reverse direction and selection effects." }, { "id": 3485, "name": "causal model", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2 ], "definition": "A representation of causal relationships and assumptions used to reason about interventions and counterfactuals. The model makes dependencies explicit; its arrows are hypotheses requiring justification, not causal facts created by drawing them." }, { "id": 3488, "name": "difference-in-differences", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 3 ], "definition": "A design comparing changes over time between treated and comparison groups. Its causal interpretation relies on assumptions such as a suitable parallel-trends counterfactual; subtracting two differences alone does not establish those assumptions." }, { "id": 3490, "name": "causal mediator", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2, 3 ], "definition": "A variable through which a proposed causal influence passes from an exposure to an outcome. Identifying mediation requires more than observing correlations among the three variables; conditioning on a mediator changes the effect being estimated." }, { "id": 4569, "name": "propensity-score matching", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "Matching treated and comparison units using their estimated treatment probability conditional on observed covariates. It seeks comparability in those covariates, under overlap and causal-identification assumptions." }, { "id": 4635, "name": "ridge regression", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Least-squares regression with a squared-coefficient-norm penalty, shrinking estimates to improve conditioning and trade bias against variance. Penalty strength and predictor scaling affect the result." }, { "id": 4678, "name": "moving-average time-series model", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A stochastic model expressing a series value through current and finitely many past innovations. It models shock propagation rather than simply averaging a sliding window of observations." }, { "id": 5505, "name": "genome-wide association study", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "An observational investigation testing many variants across genomes for statistical association with a trait, identifying associated regions without by itself establishing the causal variant, gene, or mechanism." } ] }, { "task": "A13", "split": "development", "origin": "prospective-ai-task", "domain": "mathematics", "question": "You are advising a design team minimizing a smooth cost under inequality constraints. Their solver found a feasible stationary point, but the team wants to call it the unique best design. Some constraints are active, and convexity has not been established. Explain what evidence separates feasibility, local optimality, global optimality, and uniqueness, and recommend what to verify before accepting the claim.", "requirements": [ "Distinguish feasibility, stationarity, and local optimality.", "State conditions under which multiplier or first-order arguments are justified.", "Explain what additional structure supports global optimality and uniqueness.", "Recommend checks or counterexamples that would challenge the solver's claim." ], "poolSize": 12, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 4710, 4361, 4709, 4706, 4705 ], "ndcg5": 0.8777221344374128, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.6, "embeddingRedundancy": 0.511166326775062, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 4710, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [ 0, 1, 2, 3 ] }, { "id": 4361, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [] }, { "id": 4709, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 4706, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] }, { "id": 4705, "agreedObligations": [ 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 4710, 4361, 4709, 4706, 4705 ], "ndcg5": 0.8777221344374128, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.6, "embeddingRedundancy": 0.511166326775062, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 4710, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [ 0, 1, 2, 3 ] }, { "id": 4361, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [] }, { "id": 4709, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 4706, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] }, { "id": 4705, "agreedObligations": [ 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 4709, 4361, 4705, 5738, 1464 ], "ndcg5": 0.6191922895841276, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 2 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.22574582192111664, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 4709, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 4361, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 4705, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 5738, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1464, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [ 3 ], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 1, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 4705, 4706, 4709, 4710, 4712 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.6, "embeddingRedundancy": 0.5125460348854227, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 4705, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 4706, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 4709, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 4710, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [ 0 ] }, { "id": 4712, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 792, "coverageMaximizingSubsets": 593, "coveredObligations": 4 }, "poolItems": [ { "id": 1464, "name": "continuity of a function at a point", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "For every positive output tolerance, an input neighbourhood can be chosen in which the function values stay within that tolerance of its value at the point. The requirement is local and quantified; a smooth-looking drawing alone does not establish it." }, { "id": 3352, "name": "differential geometry", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The study of geometric structures through calculus, especially on smooth manifolds. It relates local differential quantities to global geometry while retaining the assumptions that make derivatives and smooth structures available." }, { "id": 4361, "name": "mathematical optimization", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 2 ], "definition": "Selecting an optimum with respect to a stated objective over a feasible set. The objective, constraints and distinction between local and global solutions belong to the problem; mathematical optimality does not establish that the objective captures everything that matters." }, { "id": 4362, "name": "optimization problem", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "A specified task of finding a best feasible solution under an objective and constraints, with continuous or discrete decision variables. Defining the task is distinct from exhibiting a solution, proving optimality or showing that the model fits its application." }, { "id": 4702, "name": "linear programming", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "Optimizing a linear objective subject to linear equality and inequality constraints. Its feasible set is a polyhedron, which may be unbounded, empty or have multiple optimal points." }, { "id": 4705, "name": "convex optimization", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2 ], "definition": "Minimizing a convex objective over a convex feasible set, or equivalently maximizing a concave objective there. Convex structure changes the relation between local and global optimality." }, { "id": 4706, "name": "optimization duality", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "Relating a primal optimization problem to a constructed dual problem whose feasible objective values provide bounds. The problems may have different optimal values unless further conditions close the duality gap." }, { "id": 4709, "name": "Lagrange multiplier", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "An auxiliary coefficient weighting a constraint in a Lagrangian, used to characterize constrained extrema through stationarity. Valid necessity or sensitivity interpretations require suitable regularity." }, { "id": 4710, "name": "Karush–Kuhn–Tucker conditions", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2, 3 ], "definition": "Conditions combining stationarity, primal feasibility, dual feasibility and complementary slackness for constrained optimization. They are necessary under constraint qualifications and sufficient under appropriate convexity conditions." }, { "id": 4712, "name": "complementary slackness", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "A primal–dual optimality condition in which each inequality’s slack multiplied by its corresponding nonnegative multiplier is zero. An inactive inequality therefore has zero multiplier." }, { "id": 4717, "name": "Newton optimization step", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "Applying Newton root finding to the gradient of a twice-differentiable objective to seek a stationary point, using local second-derivative information. A stationary point may be a minimum, maximum or saddle." }, { "id": 5738, "name": "bacteriophage", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A virus that infects bacteria and uses a susceptible bacterial cell to reproduce its genome and produce viral progeny." } ] }, { "task": "A14", "split": "final", "origin": "prospective-ai-task", "domain": "mathematics", "question": "You are helping a campus planner connect buildings with weighted undirected links. Construction cost and emergency travel time use different edge weights, and service must continue after any single link failure. Explain why a cheapest spanning network, shortest routes, and redundancy are different objectives. Recommend a mathematical formulation and the checks needed to compare candidate designs without confusing those objectives.", "requirements": [ "Distinguish total selected-edge cost from path-distance objectives.", "Explain the connectivity property needed to survive a single edge failure.", "State why a tree-based cost solution may conflict with the resilience requirement.", "Propose a formulation or comparison procedure that keeps both weight systems explicit." ], "poolSize": 10, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 2410, 2723, 2724, 2844, 4738 ], "ndcg5": 0.7422601353304841, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.3622080350937447, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 2410, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2723, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 2724, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 2844, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 1 ] }, { "id": 4738, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 2410, 2723, 2724, 2844, 4738 ], "ndcg5": 0.7422601353304841, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.3622080350937447, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 2410, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2723, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 2724, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 2844, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 1 ] }, { "id": 4738, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 2723, 2724, 1886, 144, 2844 ], "ndcg5": 0.881237335128953, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.12181073802995053, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 2723, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 2724, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 1886, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 144, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2844, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 1 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 2723, 2724, 2844, 2721, 1886 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.5, "embeddingRedundancy": 0.226465402870572, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 2723, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 2724, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 2844, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 1 ] }, { "id": 2721, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 1886, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 252, "coverageMaximizingSubsets": 91, "coveredObligations": 4 }, "poolItems": [ { "id": 144, "name": "the outcast seen in the wilderness", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The Hagar narratives' portrayal of an afflicted enslaved woman and her child receiving divine attention beyond the central ancestral household. Seeing, hearing, naming, and provision complicate any account that limits divine concern to the preferred line alone." }, { "id": 1886, "name": "mathematical elegance as a selective cue rather than a proof", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Poincaré proposes that sensitivity to mathematical harmony helps select promising combinations for attention. An elegant candidate can still fail verification; the attraction helps explain discovery without certifying correctness." }, { "id": 2410, "name": "route length distinguished from difficulty of choosing the route", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A path can be long yet offer no consequential branching, while a shorter route can require difficult choices. Distance, decision complexity, memory requirements and guaranteed coverage measure different burdens." }, { "id": 2721, "name": "breadth-first search", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 3 ], "definition": "Exploring reachable vertices in successive layers of edge distance from a start, usually by keeping a first-in-first-out queue. In an unweighted graph this finds shortest edge-count paths; weighted costs and infinite branching require different assumptions." }, { "id": 2723, "name": "shortest-path problem", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "Finding a path between specified vertices that minimizes the sum of declared edge weights. The weights define what shortest means; the fewest edges, physical distance, time and cost are different objectives unless the model makes them equivalent." }, { "id": 2724, "name": "minimum spanning tree", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 2 ], "definition": "A cycle-free subset of a connected weighted undirected graph that includes every vertex and has the least possible total edge weight. It optimizes total connection cost, not necessarily the distance between each pair of vertices; disconnected graphs instead yield spanning forests." }, { "id": 2844, "name": "engineering redundancy", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2 ], "definition": "Deliberate provision of multiple components or ways to perform a function so that some failures need not eliminate the function. Reliability gains depend on independence and coordination; common causes and added complexity can defeat the intended protection." }, { "id": 4737, "name": "maximum network flow", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Finding the greatest feasible source-to-sink flow in a network subject to edge capacities and intermediate flow conservation. It optimizes throughput rather than route cost." }, { "id": 4738, "name": "minimum-cost network flow", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Sending a specified quantity through a directed capacitated network while minimizing the sum of edge flow multiplied by per-unit edge cost." }, { "id": 4740, "name": "bipartite graph", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A graph whose vertices can be partitioned into two disjoint parts so every edge connects vertices in different parts. For undirected graphs, this is equivalent to having no odd cycle." } ] }, { "task": "A15", "split": "final", "origin": "prospective-ai-task", "domain": "mathematics", "question": "You are preparing a calibration curve from a smooth response on a bounded interval. Measurements have small noise, and equally spaced samples currently feed a high-degree interpolating polynomial whose edge behavior is unstable. Compare exact interpolation with least-squares approximation, distinguish node placement from polynomial degree, and recommend how to balance approximation error, noise sensitivity, and trustworthy endpoint behavior.", "requirements": [ "Distinguish interpolation constraints from a least-squares fit.", "Explain separate effects of node placement and polynomial degree.", "Discuss the interaction between measurement noise and approximation stability.", "Recommend a defensible sampling and fitting strategy with appropriate limitations." ], "poolSize": 14, "poolAttainableObligations": [ 0, 2, 3 ], "poolMissingObligations": [ 1 ], "metrics": { "dense": { "top5": [ 3461, 2547, 3390, 3455, 3458 ], "ndcg5": 0.9232684181649756, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.625, "embeddingRedundancy": 0.43876213983150175, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3461, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 2547, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 3390, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 3455, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] }, { "id": 3458, "agreedObligations": [ 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3461, 2547, 3390, 3455, 3458 ], "ndcg5": 0.9232684181649756, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.625, "embeddingRedundancy": 0.43876213983150175, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3461, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 2547, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 3390, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 3455, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] }, { "id": 3458, "agreedObligations": [ 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 2547, 2219, 3461, 3597, 4627 ], "ndcg5": 0.6218131175350728, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.5, "embeddingRedundancy": 0.2799160454722512, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 2547, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [ 0, 2, 3 ] }, { "id": 2219, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3461, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 3597, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4627, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 2547, 3390, 3458, 3455, 3460 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.625, "embeddingRedundancy": 0.38984639963872125, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 2547, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [ 0, 2, 3 ] }, { "id": 3390, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 3458, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 3455, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] }, { "id": 3460, "agreedObligations": [ 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 2002, "coverageMaximizingSubsets": 1175, "coveredObligations": 3 }, "poolItems": [ { "id": 1461, "name": "declared approximation tolerance", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 3 ], "definition": "A specified positive bound on the allowable difference between an approximation and its target. Meeting one tolerance is a claim of bounded closeness, not exact equality or success under every tighter tolerance." }, { "id": 1910, "name": "an earlier theory retained as a bounded approximation", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A broader theory can reproduce an earlier theory under specified limiting conditions while predicting departures outside that approximation. Einstein gives weak fields and small speeds as conditions for recovering the Newtonian approximation." }, { "id": 2219, "name": "selecting measurements through inverse sensitivity", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "A configuration in which a small change in an unknown quantity strongly changes an observable effect can make that quantity precisely estimable. The same sensitivity that magnifies prediction error can reduce the effect of observational error in the reverse calculation." }, { "id": 2351, "name": "initial sensitivity distinguished from stability through use", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "An indicator can respond strongly to small changes yet gradually acquire an altered length, resting state or calibration. Selecting it for repeated measurement requires considering lasting drift as well as immediate sensitivity." }, { "id": 2509, "name": "an illustrative replay bounded by its reference data regime", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "A replay remains within the time interval and source model supporting its reference state, shortening or stopping at that boundary. Continuing through a fallback requires an explicit change of regime; a smooth animation must not silently turn interpolation into extrapolation or another model while retaining the original source label." }, { "id": 2547, "name": "display resolution distinguished from physical uncertainty", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 2, 3 ], "definition": "The spacing, precision or number of digits in a representation describes its numerical display. Evidence about the underlying state determines uncertainty separately. Finer grids, smaller interpolation residuals and successful coordinate round trips do not by themselves establish more accurate physical knowledge." }, { "id": 3390, "name": "numerical stability", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2 ], "definition": "The sensitivity of a computational method's output to rounding and other small computational perturbations. Stability is a property of the method and must be distinguished from an inherently ill-conditioned underlying problem." }, { "id": 3455, "name": "measurement uncertainty", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "A parameter or description characterizing the dispersion of values reasonably attributable to a measurand. It represents limits on the reported result and is not simply a known mistake to subtract from the reading." }, { "id": 3458, "name": "uncertainty propagation", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2 ], "definition": "Determining how uncertainty in inputs affects uncertainty in a derived output. Dependence between inputs and nonlinear response matter; treating every input error as independent can misstate the result." }, { "id": 3460, "name": "random measurement error", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 2 ], "definition": "A component of measurement error varying unpredictably between repeated measurements under stated conditions. Statistical treatment can characterize its variability; randomness does not imply that the complete measurement is unbiased." }, { "id": 3461, "name": "measurement calibration", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "Establishing a relation between reference values and an instrument's indications under specified conditions, including relevant uncertainties. Calibration is distinct from adjusting the instrument and does not guarantee unchanged performance in every later environment." }, { "id": 3597, "name": "thermodynamic critical point", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The endpoint of a phase-coexistence curve where the distinction between the relevant phases disappears. Large fluctuations and unusual response can occur nearby; the point is defined for a specified substance or system." }, { "id": 4627, "name": "generalized least squares", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "Linear-regression estimation accounting for a specified error covariance matrix, including correlated or unequal-variance errors. Estimating that matrix gives feasible generalized least squares with additional uncertainty." }, { "id": 4778, "name": "Bloom membership filter", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A compact probabilistic set-membership structure whose ordinary insertion-only form can report false positives but no false negatives under correct operation. It answers possibly present or definitely absent." } ] }, { "task": "A16", "split": "final", "origin": "prospective-ai-task", "domain": "mathematics", "question": "You are reviewing a linear inverse problem in which sensor readings are noisy and two physical effects produce nearly identical measurements. A fitted solution has tiny residuals but changes sharply when readings are perturbed. Distinguish uniqueness from stability and data fit from parameter accuracy. Explain what rank, singular values, and regularization can reveal, and recommend how to report the limits of reconstruction.", "requirements": [ "Distinguish an exactly nonunique inverse problem from an ill-conditioned one.", "Relate singular directions to sensitivity and identifiability.", "Explain why a small residual does not establish accurate parameters.", "Discuss a regularization choice and how its assumptions and uncertainty should be reported." ], "poolSize": 11, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 2219, 2274, 3495, 3496, 4629 ], "ndcg5": 0.9030661971488, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.3946709000102454, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 2219, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 2274, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0 ] }, { "id": 3495, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 3496, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 4629, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 2219, 2274, 3495, 3496, 4629 ], "ndcg5": 0.9030661971488, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.3946709000102454, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 2219, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 2274, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0 ] }, { "id": 3495, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 3496, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 4629, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3496, 2219, 3495, 4629, 3493 ], "ndcg5": 0.8920852691457679, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.42857142857142855, "embeddingRedundancy": 0.32566512539135195, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 3496, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 2219, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 3495, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 4629, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3493, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 2219, 2274, 3333, 3493, 3496 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 9, "labelRedundancy": 0.5555555555555556, "embeddingRedundancy": 0.2890965168234313, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 2219, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 2274, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0 ] }, { "id": 3333, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 3493, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 3496, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 462, "coverageMaximizingSubsets": 158, "coveredObligations": 4 }, "poolItems": [ { "id": 1664, "name": "religious admission detached from emancipation", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An arrangement that permits a subordinated person’s religious instruction or membership while explicitly preventing that change of status from ending their bondage." }, { "id": 2219, "name": "selecting measurements through inverse sensitivity", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "A configuration in which a small change in an unknown quantity strongly changes an observable effect can make that quantity precisely estimable. The same sensitivity that magnifies prediction error can reduce the effect of observational error in the reverse calculation." }, { "id": 2274, "name": "order ambiguity in an inverse measurement with repeated responses", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "When a response recurs at several parameter values, the response alone cannot uniquely identify the underlying value. An inverse estimate requires identifying which recurrence or order produced it, using additional evidence." }, { "id": 3333, "name": "singular value decomposition", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "A factorization of a real or complex matrix into orthogonal or unitary factors and a diagonal array of nonnegative singular values. It separates input directions, scaling and output directions, supporting rank and approximation analysis." }, { "id": 3338, "name": "system of linear equations", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0 ], "definition": "A collection of linear equations required to hold simultaneously. Depending on consistency and rank, it may have no solution, one solution or many; the number of written equations alone does not settle the outcome." }, { "id": 3368, "name": "initial value problem", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A differential equation together with specified values at an initial point or time. Those conditions select a trajectory when existence and uniqueness hypotheses hold; uncertain initial data can still yield uncertain later states." }, { "id": 3456, "name": "accuracy and precision", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "Accuracy concerns closeness to a true or reference value; precision concerns agreement among repeated results under specified conditions. Precise readings can share a substantial bias, so the two properties must not be conflated." }, { "id": 3493, "name": "statistical identifiability", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2 ], "definition": "The property that distinct model parameter values imply distinguishable observable distributions. If different parameter settings produce the same observations, more data of the same kind may not separate them without added assumptions or measurements." }, { "id": 3495, "name": "inverse problem", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 2 ], "definition": "Inferring causes, parameters or structures from observed effects through a model. Solutions may be nonunique or unstable; fitting the effects does not alone identify the generating process." }, { "id": 3496, "name": "regularization", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "Adding constraints, penalties or prior structure to stabilize estimation or restrict solutions to an ill-posed or flexible problem. It can improve useful inference while introducing assumptions that must remain visible." }, { "id": 4629, "name": "weighted least squares", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "Fitting a model by minimizing a weighted sum of squared residuals. For uncorrelated errors, inverse-variance weights support standard efficiency results when the remaining linear-model assumptions hold." } ] }, { "task": "A17", "split": "final", "origin": "prospective-ai-task", "domain": "physics", "question": "You are helping a laboratory team charge a capacitor from a fixed-voltage source through a resistor. They measure less stored capacitor energy than the energy supplied and propose using a smaller resistor to remove the discrepancy. Explain the separate roles of charge, voltage, energy, and power, compare the transient with the final state, and assess whether that change solves the claimed problem.", "requirements": [ "Set out an energy accounting across source, resistor, and capacitor.", "Distinguish charge transfer, stored energy, and instantaneous power.", "Explain how resistance affects the charging transient and total losses in the idealized setup.", "Identify what changes to the model or charging protocol would alter the conclusion." ], "poolSize": 15, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3553, 2421, 3556, 6035, 3647 ], "ndcg5": 0.6795674418464838, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.20886885821172724, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3553, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 2421, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3556, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 6035, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 3647, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3553, 2421, 3556, 6035, 3647 ], "ndcg5": 0.6795674418464838, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.20886885821172724, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3553, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 2421, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3556, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 6035, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 3647, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3647, 3553, 3511, 6035, 5934 ], "ndcg5": 0.6302530107117458, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 1, 3 ], "agreedAssignments": 3, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.18725947998410383, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3647, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3553, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 3511, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 6035, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 5934, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [ 2 ], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 1, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3511, 3553, 3556, 3694, 6000 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.42857142857142855, "embeddingRedundancy": 0.2747752506864028, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3511, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 3553, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 3556, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 3694, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0 ] }, { "id": 6000, "agreedObligations": [ 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 0 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 3003, "coverageMaximizingSubsets": 286, "coveredObligations": 4 }, "poolItems": [ { "id": 2421, "name": "correcting an accumulated offset separately from ongoing drift", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A reset removes a discrepancy already accumulated, while a change in rate or recurrence rule reduces its future growth. Either action alone can leave the other problem unresolved, so the correction should state which effect it addresses." }, { "id": 2956, "name": "mitochondrion", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A cellular organelle in most eukaryotes, with inner and outer membranes and major roles in energy metabolism, including ATP production through respiration. Its functions extend beyond a simple power-source metaphor, and abundance and structure vary with cell type and state." }, { "id": 3477, "name": "natural experiment", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A study using an externally produced allocation or change as an opportunity for causal comparison. Its credibility depends on whether the allocation supports the required assumptions, not merely on occurring outside a laboratory." }, { "id": 3511, "name": "physical power", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "The rate at which energy is transferred or converted. High power need not imply large total energy over a short interval, and a stored-energy capacity does not specify how quickly that energy can be delivered." }, { "id": 3553, "name": "capacitance", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "The relation between stored separated charge and electric potential difference for a specified configuration. It depends on geometry and materials, and characterizes electrical storage rather than a free source of energy." }, { "id": 3556, "name": "Ohm's law", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "The proportional relation between voltage and current for an ohmic element under fixed relevant conditions. Non-ohmic devices and changing temperature can violate that simple proportionality without violating charge conservation." }, { "id": 3647, "name": "electrochemistry", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The study of relations between chemical change and electrical effects, especially charge transfer at interfaces. Electrical work, reaction direction and transport constraints must be distinguished when analyzing a cell." }, { "id": 3694, "name": "internal energy", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 1 ], "definition": "A thermodynamic state quantity accounting for a system's microscopic kinetic and interaction energies within the chosen description. Its change is distinct from heat or work, which describe transfers rather than quantities stored as such." }, { "id": 5934, "name": "four-bar linkage", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A closed mechanical chain of four connected links. Joint type and geometry determine the allowed motion; planar arrangements with revolute or sliding joints are common, but spatial and spherical variants also exist." }, { "id": 6000, "name": "battery charge capacity", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "The charge a battery can deliver under a specified discharge protocol before reaching an endpoint such as a voltage limit. Ampere-hours measure charge, not energy; rate, temperature, age and cutoff conditions influence the available amount." }, { "id": 6009, "name": "AC real power", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "The average rate of net electrical energy transfer over a specified cycle or interval. It differs from apparent power and from reversible exchange with reactive elements, even when all arise from the same voltage and current waveforms." }, { "id": 6011, "name": "apparent power", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "The product of RMS voltage and RMS current for a specified AC port, expressed in volt-amperes. It describes an electrical loading scale rather than net energy transfer; adding separate apparent powers can misstate the combined load." }, { "id": 6015, "name": "electrical transformer", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A device that couples electrical circuits through changing magnetic flux, commonly to change AC voltage or provide isolation. Energy transfer requires time-varying flux; voltage transformation is distinct from creating power." }, { "id": 6035, "name": "LC resonant circuit", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "A circuit whose inductive and capacitive energy storage supports oscillatory exchange. It can select or generate a frequency response; resistance and coupling determine damping and practical bandwidth." }, { "id": 6164, "name": "power inverter", "grade": 0.5, "judgeA": 0, "judgeB": 1, "agreedObligations": [], "definition": "A device converting DC electrical input into AC output with specified voltage, frequency and waveform. It transfers and conditions supplied power rather than generating energy itself." } ] }, { "task": "A18", "split": "development", "origin": "prospective-ai-task", "domain": "physics", "question": "You are designing a classroom demonstration of a rotating space habitat. Two positions have different distances from the rotation axis, and a participant walks radially between them. Explain what a stationary participant would feel, distinguish inertial-frame acceleration from forces introduced in the rotating description, and assess how radius and rotation rate affect both apparent weight and motion-related side effects.", "requirements": [ "Relate stationary apparent weight to rotation rate and radius.", "Compare inertial and rotating-frame descriptions without double-counting forces.", "Explain the additional effects associated with radial motion.", "Discuss the tradeoff between habitat radius and rotation rate for a chosen apparent weight." ], "poolSize": 10, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3204, 1119, 5935, 3054, 1907 ], "ndcg5": 0.7495254884965651, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.42857142857142855, "embeddingRedundancy": 0.3451131654706586, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 3204, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 1119, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5935, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 3054, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 1907, "agreedObligations": [ 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3204, 1119, 5935, 3054, 1907 ], "ndcg5": 0.7495254884965651, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.42857142857142855, "embeddingRedundancy": 0.3451131654706586, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 3204, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 1119, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5935, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 3054, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 1907, "agreedObligations": [ 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3054, 3204, 5935, 4388, 6167 ], "ndcg5": 0.7866648265208778, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.18710852316904253, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3054, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 1, 2, 3 ] }, { "id": 3204, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0 ] }, { "id": 5935, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 4388, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 6167, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3054, 3204, 3518, 3683, 1907 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.5, "embeddingRedundancy": 0.32358253417974236, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3054, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 1, 2, 3 ] }, { "id": 3204, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0 ] }, { "id": 3518, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 3683, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 1907, "agreedObligations": [ 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 252, "coverageMaximizingSubsets": 56, "coveredObligations": 4 }, "poolItems": [ { "id": 1119, "name": "distinguishing force rotation from reinforcement by declared measures", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The draft asks a monitoring specification to distinguish replacement of forces from an added buildup through headcount, duration and equipment measures. A label such as rotation is insufficient by itself, while a numerical change still needs the agreed geography, time window and legitimate activity exceptions." }, { "id": 1907, "name": "one room experiment described through acceleration or gravitation", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "In Einstein’s chest illustration, an external observer explains pressure and falling objects through the room’s acceleration, while an occupant can describe them through a gravitational field. The comparison relies on the common free-fall behavior stated in the argument." }, { "id": 3054, "name": "Coriolis force", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2, 3 ], "definition": "An apparent force used to describe motion in a rotating reference frame, proportional to the object's velocity relative to that frame and the frame's rotation. It accounts for a frame-dependent deflection rather than an additional fundamental interaction." }, { "id": 3204, "name": "artificial gravity", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "Production of a sustained apparent gravitational load through acceleration, commonly rotation of a habitat. The experienced forces depend on the motion and reference frame and need not reproduce all conditions of standing on a planetary surface." }, { "id": 3518, "name": "Newtonian motion laws", "grade": 2.5, "judgeA": 3, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "The classical relations describing inertial motion, momentum change under net force, and paired interaction forces. Their ordinary form assumes an inertial frame and an appropriate classical regime; it is not an unrestricted relativistic or quantum law." }, { "id": 3683, "name": "acceleration", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "The rate of change of velocity with time in a specified frame. It includes directional change as well as change in speed, so uniform circular motion remains accelerated." }, { "id": 4388, "name": "climate-system tipping point", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A threshold beyond which part of the climate system can undergo a major self-reinforcing change under the relevant conditions. Crossing a threshold and completing the ensuing transition can occur on different timescales; the term alone provides no event forecast." }, { "id": 5935, "name": "kinematics", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "Description of motion through positions, velocities, accelerations and their relations in stated frames, without requiring an account of the forces producing it. Constraints and moving reference frames can be part of the description." }, { "id": 6167, "name": "Ekman spiral", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A depth-varying turning and weakening of current in an ideal rotating frictional boundary layer. It represents a balance involving stress and rotation, while real stratification and other forces modify the profile." }, { "id": 6172, "name": "geostrophic balance", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "An approximate horizontal balance between pressure-gradient and Coriolis forces, producing flow along pressure contours under suitable conditions. Friction, curvature, acceleration and weak rotation effects can require additional terms." } ] }, { "task": "A19", "split": "final", "origin": "prospective-ai-task", "domain": "physics", "question": "You are advising a team whose instrument is mounted on a spring and damper above a vibrating base. Increasing damping reduces ringing after a disturbance, but the team also wants to reduce vibration transmitted during continuous operation. Explain why those goals need separate frequency-dependent checks, distinguish resonance from isolation, and recommend how to evaluate a proposed change in stiffness or damping.", "requirements": [ "Distinguish a free transient from steady forced response.", "Explain how frequency ratio and damping affect resonance and transmission.", "Assess why a change that improves ringing may affect isolation differently.", "Recommend measurements or a model comparison before selecting stiffness and damping." ], "poolSize": 16, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 5947, 1234, 5950, 2850, 5958 ], "ndcg5": 0.9142126528861984, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.26954469621887783, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 5947, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 1234, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 5950, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 2850, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 5958, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 5947, 1234, 5950, 2850, 5958 ], "ndcg5": 0.9142126528861984, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.26954469621887783, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 5947, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 1234, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 5950, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 2850, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 5958, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 5950, 5958, 5947, 1917, 4470 ], "ndcg5": 0.6595498249655878, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.0, "embeddingRedundancy": 0.12074789067474408, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 5950, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 5958, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5947, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 1917, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4470, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 5947, 5950, 1234, 2021, 2850 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.42857142857142855, "embeddingRedundancy": 0.3446812500894953, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 5947, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 5950, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 1234, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 2021, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 2850, "agreedObligations": [ 0 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 4368, "coverageMaximizingSubsets": 364, "coveredObligations": 4 }, "poolItems": [ { "id": 1234, "name": "control of extraneous experimental disturbance", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "An experimental practice that limits disturbances unrelated to the question being examined, so variation in the observed result can be interpreted against a more stable set of conditions." }, { "id": 1917, "name": "desire and capacity as a proposed balance", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Rousseau’s proposed measure of well-being compares the reach of desire with available capacity. Increasing capacity or reducing desire in isolation need not produce the balance his account seeks." }, { "id": 2021, "name": "control response matched to disturbance timescale", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "A regulating mechanism must respond on a timescale appropriate to the disturbances it is meant to limit. Stored inertia that softens a change in one system may be absent in another, requiring a quicker control response." }, { "id": 2145, "name": "a production stoppage transmitted through reduced demand", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An interruption in one connected activity reduces its participants’ ability to buy elsewhere, transmitting difficulty across a network of production and exchange. Interdependence can spread a disturbance beyond the point where it began." }, { "id": 2259, "name": "an upstream medium limiting an ideal instrument", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A disturbance imposed before a signal reaches an instrument can limit performance even if the instrument itself is perfectly made. Improving internal accuracy cannot alone recover distinctions already confused during propagation." }, { "id": 2850, "name": "oscillation", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "Repeated variation of a quantity around a central value or between states. Oscillations can be sustained, damped or driven and occur in many kinds of systems; recognizing a cycle does not identify its mechanism or make its future timing certain." }, { "id": 3479, "name": "observational study", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A study in which investigators observe exposures or conditions without assigning them as experimental interventions. Associations can inform causal inquiry, but causal interpretation requires assumptions about selection, measurement and alternative causes." }, { "id": 3675, "name": "frequency", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "The number of repetitions of a periodic event per unit time, or a specified rate of occurrence. The counted event and time reference must be stated; frequency is not the same as the speed at which a pattern travels through space." }, { "id": 4470, "name": "covering space", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A space equipped with a continuous map to a base space such that each base point has a neighborhood whose preimage separates into open sheets, each mapped homeomorphically onto that neighborhood. Local copies need not assemble into a globally trivial product." }, { "id": 5751, "name": "frequency-dependent selection", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Selection in which a genotype or phenotype fitness depends on its frequency or the composition of the surrounding population, with commonness potentially increasing or decreasing its advantage." }, { "id": 5947, "name": "oscillation damping", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 0, 2 ], "definition": "Loss of organized oscillatory energy through dissipative processes such as friction or electrical resistance. It can reduce oscillation amplitude or suppress oscillatory return; a velocity-proportional damping force is one model rather than a definition of all damping." }, { "id": 5950, "name": "mechanical impedance", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "The frequency-dependent relation between applied harmonic force and resulting velocity at a stated mechanical point or port. It includes both magnitude and phase, and is distinct from static stiffness." }, { "id": 5958, "name": "room acoustics", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The study of how enclosed or partly enclosed spaces shape sound through reflection, absorption, diffraction and modal behavior. Geometry and surfaces make the response vary with frequency and location rather than producing one uniform acoustic condition." }, { "id": 5959, "name": "ultrasound", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Sound above the conventional upper frequency range of human hearing, usually about 20 kilohertz. It remains a mechanical wave in a medium and can support ranging, imaging or processing; the frequency label alone establishes neither safety nor diagnostic accuracy." }, { "id": 6035, "name": "LC resonant circuit", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A circuit whose inductive and capacitive energy storage supports oscillatory exchange. It can select or generate a frequency response; resistance and coupling determine damping and practical bandwidth." }, { "id": 6191, "name": "ice–albedo feedback", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A feedback in which changing snow or ice cover alters reflected sunlight, influencing temperature and further cover change. Loss of reflective cover can amplify warming; gain can amplify cooling under suitable conditions." } ] }, { "task": "A20", "split": "final", "origin": "prospective-ai-task", "domain": "physics", "question": "You are helping an instrument team whose device operates in vacuum near a warm surface. They propose a shiny shield and assume that eliminating air eliminates heat transfer. Explain the roles of radiation, residual conduction through supports, and thermal storage. Distinguish equilibrium temperature from the time required to approach it, and recommend which properties and measurements determine whether the shield helps.", "requirements": [ "Identify the heat-transfer paths that remain in vacuum.", "Distinguish radiative properties and geometry from conductive paths.", "Separate steady energy balance from heat-capacity-controlled transients.", "Recommend measurements or model inputs needed to evaluate the shield." ], "poolSize": 17, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 2102, 3189, 5965, 6191, 3579 ], "ndcg5": 0.5464913089438267, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.625, "embeddingRedundancy": 0.34668369506695745, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 2102, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 3189, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 5965, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0 ] }, { "id": 6191, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3579, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 2102, 3189, 5965, 6191, 3579 ], "ndcg5": 0.5464913089438267, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.625, "embeddingRedundancy": 0.34668369506695745, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 2102, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 3189, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 5965, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0 ] }, { "id": 6191, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3579, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 5965, 4394, 2102, 5563, 3189 ], "ndcg5": 0.6820544043577793, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.5, "embeddingRedundancy": 0.26084469065134286, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 5965, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 4394, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2 ] }, { "id": 2102, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] }, { "id": 5563, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3189, "agreedObligations": [ 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 }, "oracle": { "top5": [ 3563, 3575, 3577, 3578, 3579 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 14, "labelRedundancy": 0.7142857142857143, "embeddingRedundancy": 0.4032111004175521, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3563, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 1, 2 ] }, { "id": 3575, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3577, "agreedObligations": [ 0, 1, 2, 3 ], "newObligationsAtRank": [] }, { "id": 3578, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [] }, { "id": 3579, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 6188, "coverageMaximizingSubsets": 4670, "coveredObligations": 4 }, "poolItems": [ { "id": 1262, "name": "thermal containment for a high-temperature process", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An arrangement that limits heat transfer away from an active region so a required high temperature can be maintained more effectively within it." }, { "id": 2102, "name": "a cover’s effect reversed by its thickness", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 3 ], "definition": "A covering that can aid transfer of heat when thin but inhibit it when thick enough, changing its effect on the underlying material. Surface absorption and transmission through the cover are distinct contributions." }, { "id": 3189, "name": "spacecraft thermal control", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "Management of heat generation, transfer, absorption and rejection to keep spacecraft components within their required temperature ranges. It addresses both environmental exposure and internally generated heat throughout mission conditions." }, { "id": 3562, "name": "temperature", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "An intensive thermodynamic quantity characterizing thermal state and the direction of spontaneous heat transfer between systems. It is not the total heat or energy contained in a body." }, { "id": 3563, "name": "heat as energy transfer", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2 ], "definition": "Energy transferred because of a temperature difference through conduction, convection or radiation. Heat names transfer rather than a substance stored inside a body; internal energy is a state quantity, and rates depend on material and boundary conditions." }, { "id": 3564, "name": "thermal equilibrium", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "A condition in which systems in thermal contact have no net heat transfer between them and share a temperature under the applicable thermodynamic description. Other kinds of equilibrium need not follow." }, { "id": 3575, "name": "heat capacity", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "The heat transfer needed per unit temperature change for a body under a specified process constraint. Constant-pressure and constant-volume capacities can differ; capacity scales with the amount of material." }, { "id": 3577, "name": "thermal conductivity", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2, 3 ], "definition": "A material property relating conductive heat flux to a temperature gradient in a specified model. It can depend on direction, temperature and composition, and differs from heat capacity." }, { "id": 3578, "name": "thermal radiation", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 3 ], "definition": "Electromagnetic radiation emitted because of matter's thermal state. Its spectrum and power depend on temperature and emissive properties; radiation does not require a material medium between emitter and receiver." }, { "id": 3579, "name": "black-body radiation", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "The equilibrium electromagnetic emission of an ideal perfect absorber, with a spectrum determined by temperature. Real surfaces may depart from this ideal through wavelength- and direction-dependent emissivity." }, { "id": 4394, "name": "thermal diffusivity", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "Thermal conductivity divided by volumetric heat capacity under the stated material assumptions, describing how rapidly temperature differences diffuse through a medium. Conductivity and heat storage jointly matter; good conduction alone does not specify the transient response." }, { "id": 4395, "name": "Biot number", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "A dimensionless comparison of resistance to heat conduction inside a body with resistance to heat transfer at its surface. A small value can support an approximately uniform internal-temperature model; that approximation still depends on geometry and boundary conditions." }, { "id": 5563, "name": "chemiosmosis", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Coupling of ion movement down an electrochemical gradient across a membrane to useful cellular work, notably ATP synthesis driven by proton flow through ATP synthase." }, { "id": 5965, "name": "thermal contact conductance", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 3 ], "definition": "A measure of heat transfer across the interface between touching bodies for a stated interface temperature difference. Imperfect contact can sustain a temperature drop even when the adjacent materials each conduct heat well." }, { "id": 5995, "name": "compressed-air energy storage", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Storage that compresses air for later expansion and energy recovery. Compression heat may be retained, rejected and later replaced, or exchanged during near-isothermal operation; supplementary heat belongs in the energy account." }, { "id": 5998, "name": "seasonal thermal storage", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "Thermal storage spanning seasonal mismatches between availability and demand, such as summer heat used in winter. It extends the time horizon of storage, making long-duration losses, temperature levels and site conditions consequential." }, { "id": 6191, "name": "ice–albedo feedback", "grade": 0.5, "judgeA": 0, "judgeB": 1, "agreedObligations": [], "definition": "A feedback in which changing snow or ice cover alters reflected sunlight, influencing temperature and further cover change. Loss of reflective cover can amplify warming; gain can amplify cooling under suitable conditions." } ] }, { "task": "A21", "split": "final", "origin": "prospective-ai-task", "domain": "statistics", "question": "You are reviewing a team's randomized website experiment. They compare two designs on three outcome measures, inspect results every day, and want to stop as soon as any conventional significance threshold is crossed. Explain how repeated looks and multiple outcomes differ, separate statistical evidence from practical effect size, and recommend a decision procedure that preserves an interpretable error guarantee.", "requirements": [ "Distinguish multiplicity across outcomes from repeated sequential looks.", "Explain how the proposed stopping rule affects an ordinary fixed-sample analysis.", "Incorporate effect size and uncertainty into the decision goal.", "Recommend a prespecified procedure with clear error control and stopping rules." ], "poolSize": 15, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3449, 3472, 3475, 3478, 4699 ], "ndcg5": 0.4710468685170683, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 3 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.4316145109250443, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 3449, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 3472, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3475, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3478, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4699, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3472, 3449, 3475, 3478, 4699 ], "ndcg5": 0.348585852916874, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 3 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.4316145109250443, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 3472, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3449, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 3475, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3478, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4699, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 4684, 3475, 3449, 147, 4288 ], "ndcg5": 0.6155580477282292, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.4, "embeddingRedundancy": 0.12382569272466142, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 4684, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 3475, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3449, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 147, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4288, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 1 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 }, "oracle": { "top5": [ 3429, 3449, 4684, 3428, 3426 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 9, "labelRedundancy": 0.5555555555555556, "embeddingRedundancy": 0.3397609153127689, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3429, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3449, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0 ] }, { "id": 4684, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 3428, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2 ] }, { "id": 3426, "agreedObligations": [ 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 1, 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 2 } }, "oracleEnumeration": { "subsetsEnumerated": 3003, "coverageMaximizingSubsets": 286, "coveredObligations": 4 }, "poolItems": [ { "id": 147, "name": "the binding of Isaac", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The narrative of Abraham commanded to offer his son, brought to the point of sacrifice, and then stopped, with a ram supplied instead. It concentrates promise, obedience, danger, and substitution in a scene whose moral and theological implications remain contested." }, { "id": 2750, "name": "eventual consistency", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A replication guarantee that, if further updates stop and the required communication and recovery conditions hold, replicas converge to a common resolved state. It permits temporary disagreement and does not by itself bound how soon convergence occurs or what intermediate reads observe." }, { "id": 3426, "name": "statistical hypothesis test", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "A decision procedure comparing observations with expectations under a specified statistical hypothesis. Rejection or non-rejection depends on assumptions and a decision rule; failing to reject does not establish that the hypothesis is true." }, { "id": 3428, "name": "statistical power", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "The probability that a test rejects its null hypothesis for a specified alternative when that alternative holds. Power depends on effect, design, sample size, noise and decision threshold; it is not a property of sample size alone." }, { "id": 3429, "name": "statistical decision errors", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "A type I error rejects a true null hypothesis; a type II error fails to reject it under a specified alternative. Their probabilities belong to the chosen hypotheses and decision procedure, rather than a general measure of factual correctness." }, { "id": 3449, "name": "multiple comparisons problem", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "The increased opportunity for misleading findings when many statistical comparisons are made and interpreted selectively. Error control depends on the family of questions and procedure, not only the threshold for each individual test." }, { "id": 3450, "name": "meta-analysis", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Statistical synthesis of results from multiple studies addressing related questions. Validity depends on study compatibility, bias, uncertainty and selection; combining estimates does not automatically remove weaknesses shared by the studies." }, { "id": 3460, "name": "random measurement error", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A component of measurement error varying unpredictably between repeated measurements under stated conditions. Statistical treatment can characterize its variability; randomness does not imply that the complete measurement is unbiased." }, { "id": 3472, "name": "experiment", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An inquiry that deliberately controls or changes conditions to examine resulting observations against a question or hypothesis. Its inferential strength depends on design, comparison and measurement, not merely on intervening." }, { "id": 3475, "name": "randomized controlled trial", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An experiment assigning eligible units to interventions by a random mechanism and comparing outcomes. Randomization limits certain confounding in expectation, but attrition, nonadherence, measurement and limited scope can still affect interpretation." }, { "id": 3478, "name": "quasi-experiment", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An intervention study without full random assignment that uses design or analysis to support a causal comparison. Its identifying assumptions require explicit scrutiny because treatment groups may differ for other reasons." }, { "id": 4189, "name": "hierarchy of evidence", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A heuristic ordering of study designs or evidence types by their expected strength for a specified question, often by susceptibility to bias. The ranking depends on the question, outcome and execution; a high-ranked design is not a guarantee that a particular result is sound." }, { "id": 4288, "name": "Buddhist two-truths doctrine", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A distinction between conventional and ultimate truth or reality in Buddhist teaching. Schools differ about how the two relate; distinguishing levels does not automatically license any contradiction or make conventional concerns disposable." }, { "id": 4684, "name": "sequential statistical analysis", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 3 ], "definition": "Evaluating accumulating data under a prespecified monitoring and stopping procedure, allowing sample size to depend on observations while accounting for the repeated decisions." }, { "id": 4699, "name": "experimental blocking", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Grouping experimental units by relevant nuisance variation and arranging treatment comparisons within those groups. The purpose is to separate the treatment comparison from known sources of variability." } ] }, { "task": "A22", "split": "development", "origin": "prospective-ai-task", "domain": "statistics", "question": "You are advising a researcher comparing repeated survey scores between two programs. Participants with worse earlier scores are more likely to miss later visits, and the researcher suspects unrecorded current scores also affect attendance. Compare complete-case analysis, weighting, and imputation under explicit missingness assumptions. Recommend an analysis and sensitivity checks that separate what the observed data support from what must be assumed.", "requirements": [ "Distinguish missingness explained by observed history from dependence on unobserved outcomes.", "State assumptions for complete-case, weighting, and imputation approaches.", "Recommend a primary analysis suited to the recorded longitudinal information.", "Specify sensitivity analyses and reporting for unverifiable missingness assumptions." ], "poolSize": 12, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 4668, 4665, 3480, 4578, 3448 ], "ndcg5": 0.7446551257189392, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 9, "labelRedundancy": 0.5555555555555556, "embeddingRedundancy": 0.38340879627793106, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 4668, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 4665, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0 ] }, { "id": 3480, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2 ] }, { "id": 4578, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 3448, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 4668, 4665, 3480, 4578, 3448 ], "ndcg5": 0.7446551257189392, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 9, "labelRedundancy": 0.5555555555555556, "embeddingRedundancy": 0.38340879627793106, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 4668, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 4665, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0 ] }, { "id": 3480, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2 ] }, { "id": 4578, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 3448, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 4665, 3480, 4668, 3447, 3477 ], "ndcg5": 0.746517056925071, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.42857142857142855, "embeddingRedundancy": 0.2192616545512355, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 4665, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 3480, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 4668, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] }, { "id": 3447, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 3477, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3447, 4665, 4666, 4667, 4668 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 11, "labelRedundancy": 0.6363636363636364, "embeddingRedundancy": 0.48141634948479134, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3447, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 4665, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 4666, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 4667, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [] }, { "id": 4668, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 792, "coverageMaximizingSubsets": 627, "coveredObligations": 4 }, "poolItems": [ { "id": 3439, "name": "analysis of variance", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Methods that partition observed variability into components to compare effects or group means under a statistical model. The resulting tests depend on the design and error assumptions, not solely on the arithmetic partition." }, { "id": 3447, "name": "sensitivity analysis", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "Examining how a model's outputs change when inputs or assumptions vary. It locates dependence within a model but does not establish that the varied ranges exhaust real uncertainty." }, { "id": 3448, "name": "missing data", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 1 ], "definition": "Values absent for some variables or units intended for analysis. The mechanism producing absence affects which inferences are justified; an unrecorded value is not generally equivalent to zero or a negative finding." }, { "id": 3477, "name": "natural experiment", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A study using an externally produced allocation or change as an opportunity for causal comparison. Its credibility depends on whether the allocation supports the required assumptions, not merely on occurring outside a laboratory." }, { "id": 3480, "name": "longitudinal study", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "A study following units through multiple times to examine change or temporal relationships. Repeated observation helps describe trajectories but does not alone eliminate confounding, attrition or measurement changes." }, { "id": 4192, "name": "intention-to-treat analysis", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "Analyzing randomized participants according to their assigned groups regardless of subsequent treatment received or adherence. It preserves the assignment-based comparison, while missing outcomes and other design problems still require explicit assumptions and handling." }, { "id": 4578, "name": "paired-difference analysis", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "Comparing conditions through differences within matched pairs or repeated measurements, retaining the dependence between paired observations rather than treating them as independent groups." }, { "id": 4665, "name": "missing completely at random", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "A missingness mechanism independent of both observed and unobserved study values. It is stronger than missing at random and can justify certain complete-case analyses under the remaining model assumptions." }, { "id": 4666, "name": "missing at random", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2 ], "definition": "A missingness mechanism that, conditional on the relevant observed values, does not additionally depend on the missing values. It is an assumption supporting specified likelihood or imputation procedures." }, { "id": 4667, "name": "missing not at random", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 3 ], "definition": "A missingness mechanism that still depends on unobserved values after conditioning on the relevant observed information. Valid inference then needs further modeling, identifying assumptions or sensitivity analysis." }, { "id": 4668, "name": "statistical imputation", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "Replacing missing values by values generated or estimated under a stated procedure. Imputed values support analysis but remain inferred substitutions rather than recovered observations." }, { "id": 4669, "name": "multiple imputation", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2 ], "definition": "Generating multiple plausible completed datasets, analyzing each and combining results to reflect uncertainty from missing values as well as ordinary sampling uncertainty." } ] }, { "task": "A23", "split": "final", "origin": "prospective-ai-task", "domain": "statistics", "question": "You are evaluating whether a cheaper sensor can replace a reference device. Paired readings have very high correlation, but the difference between devices grows with the reading level and each object is measured several times. Explain the difference between association and agreement, distinguish systematic bias from random error, and propose a validation analysis tied to an acceptable replacement tolerance.", "requirements": [ "Explain why high correlation alone does not establish interchangeability.", "Assess constant or magnitude-dependent bias separately from random differences.", "Account for repeated measurements on the same objects.", "Recommend an agreement assessment and decision criterion linked to a stated tolerance." ], "poolSize": 12, "poolAttainableObligations": [ 1, 2, 3 ], "poolMissingObligations": [ 0 ], "metrics": { "dense": { "top5": [ 4197, 3456, 3461, 1882, 3492 ], "ndcg5": 0.8114968423174953, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.7, "embeddingRedundancy": 0.4596536434436551, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 4197, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 3456, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 3461, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] }, { "id": 1882, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] }, { "id": 3492, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 4197, 3456, 3461, 1882, 3492 ], "ndcg5": 0.8114968423174953, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.7, "embeddingRedundancy": 0.4596536434436551, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 4197, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 3456, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 3461, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] }, { "id": 1882, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] }, { "id": 3492, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 4197, 4641, 4608, 3492, 509 ], "ndcg5": 0.3288361908386545, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.14918537367801082, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 4197, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 4641, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4608, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3492, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 509, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 1882, 3456, 3459, 3461, 4578 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.7, "embeddingRedundancy": 0.427916111328038, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 1882, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 1, 2 ] }, { "id": 3456, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3459, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] }, { "id": 3461, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] }, { "id": 4578, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 792, "coverageMaximizingSubsets": 696, "coveredObligations": 3 }, "poolItems": [ { "id": 509, "name": "intercession bounded by divine permission", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The Qur'anic limitation that intercession at the reckoning benefits only under the Merciful's permission and approval. Another's intervention is not presented as an independently controllable route around divine judgment." }, { "id": 1882, "name": "systematic error distinguished from averaging variation", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2 ], "definition": "Repeated readings may reduce the influence of variable error while leaving a shared directional error intact. A systematic correction requires investigating its cause rather than assuming an average removes it." }, { "id": 2254, "name": "a law measured on a mixture requiring component-level validation", "grade": 0.5, "judgeA": 0, "judgeB": 1, "agreedObligations": [], "definition": "A relation measured on an unresolved mixture may describe its middle or dominant contribution without establishing the same relation for every constituent. Extending it to separated components requires additional evidence." }, { "id": 3456, "name": "accuracy and precision", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "Accuracy concerns closeness to a true or reference value; precision concerns agreement among repeated results under specified conditions. Precise readings can share a substantial bias, so the two properties must not be conflated." }, { "id": 3459, "name": "systematic measurement error", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2 ], "definition": "A component of measurement error that remains constant or changes predictably across repeated measurements under specified conditions. Repetition alone need not remove it; correction requires a model or comparison that identifies the bias." }, { "id": 3461, "name": "measurement calibration", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "Establishing a relation between reference values and an instrument's indications under specified conditions, including relevant uncertainties. Calibration is distinct from adjusting the instrument and does not guarantee unchanged performance in every later environment." }, { "id": 3492, "name": "inter-rater reliability", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 3 ], "definition": "Agreement or consistency among assessors rating the same units under a stated measure. Raters can agree while sharing a systematic error, and agreement statistics depend on categories, prevalence and comparison assumptions." }, { "id": 4197, "name": "measurement repeatability", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "Closeness of agreement among repeated measurements of the same measurand under stated repeatability conditions, such as the same procedure, observer, instrument, location and short time interval. Agreement under these conditions does not establish accuracy or performance under changed conditions." }, { "id": 4562, "name": "systematic sampling", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Selecting units from an ordered sampling frame at a regular interval after a random start. Its performance depends on how the frame’s ordering relates to the measured quantity." }, { "id": 4578, "name": "paired-difference analysis", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "Comparing conditions through differences within matched pairs or repeated measurements, retaining the dependence between paired observations rather than treating them as independent groups." }, { "id": 4608, "name": "bias–variance trade-off", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "The tension between systematic prediction error and sensitivity to the training sample when selecting a modeling procedure. Greater flexibility can reduce bias while increasing variance, depending on the model and regime." }, { "id": 4641, "name": "canonical correlation analysis", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Finding linear combinations of two sets of variables whose correlation is maximal, with further pairs constrained relative to earlier ones. It characterizes cross-set linear association." } ] }, { "task": "A24", "split": "final", "origin": "prospective-ai-task", "domain": "statistics", "question": "You are reviewing a survey that oversampled small districts and interviewed several people within each selected household. The team proposes an unweighted average and an ordinary independent-observation standard error. Explain how unequal selection probabilities and within-household dependence affect different parts of the estimate, distinguish descriptive population targets from the realized sample, and recommend the information needed for a defensible estimate and uncertainty interval.", "requirements": [ "Define the population quantity and distinguish it from an unweighted sample summary.", "Explain the role of selection probabilities or design weights in estimation.", "Explain how household clustering affects uncertainty calculation.", "Identify the design information and any nonresponse adjustments needed for analysis." ], "poolSize": 14, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3432, 4565, 4564, 4563, 3442 ], "ndcg5": 0.6285608281560209, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.2, "embeddingRedundancy": 0.4690671117472881, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3432, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 4565, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4564, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 4563, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3442, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3432, 4565, 4564, 4563, 3442 ], "ndcg5": 0.6285608281560209, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.2, "embeddingRedundancy": 0.4690671117472881, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3432, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 4565, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4564, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 4563, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3442, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3431, 4563, 3458, 3432, 4156 ], "ndcg5": 0.7638644618689432, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.5, "embeddingRedundancy": 0.21553993800945856, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 3431, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 4563, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] }, { "id": 3458, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 3432, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 4156, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 4557, 4563, 3431, 3432, 3458 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.6, "embeddingRedundancy": 0.34970067393201704, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 4557, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 4563, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 3431, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0 ] }, { "id": 3432, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 3458, "agreedObligations": [ 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 2002, "coverageMaximizingSubsets": 734, "coveredObligations": 4 }, "poolItems": [ { "id": 3431, "name": "statistical sampling", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 0, 1, 3 ], "definition": "Selecting a subset of units from a defined population to obtain information about it. The sampling frame, selection mechanism and target population determine which inferences the sample can support." }, { "id": 3432, "name": "sampling bias", "grade": 2.5, "judgeA": 3, "judgeB": 2, "agreedObligations": [ 0, 1 ], "definition": "A systematic mismatch between a sample's selection process and the population or quantity intended for inference. Increasing sample size need not repair that mismatch." }, { "id": 3434, "name": "survivorship bias", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A selection bias in which surviving or remaining cases receive attention while lost or excluded cases are absent. Apparent success among survivors can therefore misrepresent the process that produced both survivors and failures." }, { "id": 3442, "name": "bootstrap resampling", "grade": 1.5, "judgeA": 2, "judgeB": 1, "agreedObligations": [], "definition": "Estimating a statistic's variability by repeatedly resampling from an empirical sample under a specified scheme. The procedure inherits limitations of the sample and resampling model; it does not create missing population information." }, { "id": 3458, "name": "uncertainty propagation", "grade": 2.5, "judgeA": 3, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "Determining how uncertainty in inputs affects uncertainty in a derived output. Dependence between inputs and nonlinear response matter; treating every input error as independent can misstate the result." }, { "id": 4156, "name": "topic map", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A knowledge-organization representation distinguishing topics, associations between topics and occurrences that connect topics to information resources. Types, scoped names and subject identifiers support organization and merging under declared rules." }, { "id": 4557, "name": "cluster sampling", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "Selecting groups from a population, then observing all or a sample of the units within selected groups. The design trades collection cost against dependence and variation within and between clusters." }, { "id": 4563, "name": "design-based survey inference", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "Inference about a target population based on the probabilities specified by a survey sampling design. The design supplies a sampling distribution but does not eliminate measurement, nonresponse or coverage bias." }, { "id": 4564, "name": "standard error", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "The standard deviation of an estimator’s sampling distribution. It describes variability of the estimate under repeated sampling, rather than the variability of individual observations." }, { "id": 4565, "name": "finite-population correction", "grade": 1.5, "judgeA": 2, "judgeB": 1, "agreedObligations": [], "definition": "Adjustment of sampling uncertainty for sampling without replacement from a fixed finite population. Sampling a larger fraction reduces remaining sampling uncertainty relative to an otherwise comparable with-replacement calculation." }, { "id": 4579, "name": "Welch unequal-variance test", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An approximate two-sample test of mean equality that estimates variability separately for independent groups rather than assuming equal population variances." }, { "id": 4693, "name": "statistical tolerance interval", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "A sample-based interval designed, at a stated confidence level, to contain at least a specified fraction of a population. Population content and confidence are separate parameters." }, { "id": 4766, "name": "Fano error–uncertainty bound", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An information-theoretic inequality connecting uncertainty remaining about a discrete input after observation with the error probability of estimating it. It supports lower bounds on achievable decoding or estimation error." }, { "id": 5253, "name": "electoral cracking", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A districting tactic dispersing a group's voters across several districts so that their voting strength is diluted within each. It is distinct from concentrating the group in a small number of districts." } ] }, { "task": "B01", "split": "final", "origin": "public-human-question", "domain": "philosophy", "question": "How does moral luck challenge the Kantian claim that moral worth depends on a good will? Compare the role of intention with factors outside an agent's control, and distinguish the different kinds of luck named in the question. Explain whether the objection concerns moral worth itself or our assessment of an agent.", "requirements": [ "Explain the Kantian claim about good will and acting from duty.", "Distinguish at least two of the named kinds of moral luck.", "Show how lack of control could challenge the stated account of moral worth.", "Separate the philosophical objection from a mere difference in everyday judgments." ], "poolSize": 14, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3836, 2003, 3832, 3938, 2012 ], "ndcg5": 0.6978872952515099, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.4227584994877679, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3836, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 1, 2, 3 ] }, { "id": 2003, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [ 0 ] }, { "id": 3832, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3938, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2012, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3836, 2003, 3832, 3938, 2012 ], "ndcg5": 0.6978872952515099, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.4227584994877679, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3836, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 1, 2, 3 ] }, { "id": 2003, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [ 0 ] }, { "id": 3832, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3938, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2012, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3836, 1663, 2003, 5008, 1941 ], "ndcg5": 0.6175382652479369, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.16885637281656996, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3836, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 1, 2, 3 ] }, { "id": 1663, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2003, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [ 0 ] }, { "id": 5008, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1941, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 1415, 2003, 3836, 3820, 2182 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 11, "labelRedundancy": 0.6363636363636364, "embeddingRedundancy": 0.34309081173569506, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 1415, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 1, 2, 3 ] }, { "id": 2003, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [ 0 ] }, { "id": 3836, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [] }, { "id": 3820, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] }, { "id": 2182, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 2002, "coverageMaximizingSubsets": 385, "coveredObligations": 4 }, "poolItems": [ { "id": 1415, "name": "outcomes altering perceived desert", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2, 3 ], "definition": "A pattern in which the result of an action changes how much gratitude, blame or punishment it attracts, even when the agent’s intention or care is held similar. What was controlled by the agent and what happened through fortune remain separate questions." }, { "id": 1663, "name": "an observed analogue that removes an impossibility objection", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An observed configuration used to show that an alleged structural impossibility is realizable, without by itself proving every claim in the broader theory the objection targeted." }, { "id": 1788, "name": "receiving support without injuring its source", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The bee-and-flower image presents taking sustenance while leaving the source’s form and qualities unharmed as a model for a sage’s presence in a community. The analogy guides conduct without proving that every exchange is harmless." }, { "id": 1941, "name": "affection offered as an answer to objection", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Responding to a substantive objection with reassurance of love, sacrifice or benevolent intention, without resolving the grounds of the objection. Affection can be real while remaining insufficient as an answer." }, { "id": 2003, "name": "Kantian good will independent of achieved success", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 2, 3 ], "definition": "Kant’s proposed unconditional worth of willing rightly, distinguished from success, usefulness and fortune. It requires summoning available means, not merely wishing for a good outcome; the theory’s evaluation of willing remains distinct from an account of consequences." }, { "id": 2012, "name": "dignity distinguished from exchangeable value", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Kant’s distinction between what admits an equivalent replacement and the intrinsic worth he assigns to rational moral agency. Dignity in this account is not a higher market price or a measurement of usefulness." }, { "id": 2182, "name": "moral censure calibrated to a different audience", "grade": 1.5, "judgeA": 2, "judgeB": 1, "agreedObligations": [], "definition": "Conduct tolerated in one setting may be presented as exceptionally discrediting to another audience whose standards make the charge useful. Examining that selective appeal does not by itself settle whether the conduct occurred or whether either standard is justified." }, { "id": 3820, "name": "moral responsibility", "grade": 2.5, "judgeA": 3, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "A status in virtue of which an agent can appropriately receive moral appraisal for an action or omission. It is distinguished from causal involvement and legal liability; accounts differ about how control, knowledge and freedom bear on its attribution." }, { "id": 3832, "name": "virtue ethics", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Ethical approaches that give central importance to character, dispositions and practical excellence in living and acting. Evaluating one outcome or outwardly correct act is distinguished from understanding the qualities through which an agent habitually perceives and responds." }, { "id": 3836, "name": "moral luck", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2, 3 ], "definition": "Dependence of moral appraisal on factors beyond an agent's control, including outcomes, encountered circumstances or aspects of the agent's formation. The resulting tension with control-based responsibility is a philosophical problem, not a settled rule excusing or blaming everyone alike." }, { "id": 3847, "name": "Gettier problem", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "The challenge that a belief can be true and apparently justified yet fall short of knowledge because its truth depends on epistemic luck. Gettier-style cases test whether justified true belief supplies sufficient conditions for knowledge rather than denying that justification matters." }, { "id": 3938, "name": "moral hazard", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An incentive to change behavior when another party bears some of the consequences, particularly when actions are difficult to observe or constrain. The problem concerns altered incentives after risk-sharing or delegation, not a necessary judgment about an actor's moral character." }, { "id": 5008, "name": "peritext", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "In Genette's account of paratext, framing elements presented with the work itself, such as its title, preface or notes. Their location and role distinguish them from associated discussion circulating outside the publication." }, { "id": 5101, "name": "public ownership", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Ownership of assets or enterprises by a government or public body at a specified level. It concerns who owns or controls the assets, distinct from whether their outputs are public goods or whether management is democratic, commercial or effective." } ] }, { "task": "B02", "split": "development", "origin": "public-human-question", "domain": "philosophy", "question": "How should induction and falsifiability be related when judging the claims that I am mortal or immortal? Assume the past deaths cited in the question, and set aside biological mechanisms. Explain the difference between evidence for believing a claim and a criterion for testing a hypothesis, and whether either method should take priority here.", "requirements": [ "Distinguish support from past observations from a hypothesis's testability.", "Compare the mortal and immortal claims from the question.", "Respect the restriction to induction rather than appealing to biological mechanisms.", "Explain what would and would not follow from preferring a falsifiable hypothesis." ], "poolSize": 14, "poolAttainableObligations": [ 0, 2, 3 ], "poolMissingObligations": [ 1 ], "metrics": { "dense": { "top5": [ 3464, 3248, 1017, 1570, 2525 ], "ndcg5": 0.9648401609087868, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.4, "embeddingRedundancy": 0.29571089409104107, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3464, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 3248, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 1017, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1570, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 2525, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3464, 3248, 1017, 1570, 2525 ], "ndcg5": 0.9648401609087868, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.4, "embeddingRedundancy": 0.29571089409104107, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3464, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 3248, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 1017, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1570, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 2525, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3464, 1017, 2105, 2525, 3379 ], "ndcg5": 0.5721175411760587, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 3 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.22883713671033362, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 3464, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 1017, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2105, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2525, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3379, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [ 2 ], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 1, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3248, 3464, 1570, 4189, 358 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.4, "embeddingRedundancy": 0.28030647833383987, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3248, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 3464, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 1570, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 4189, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 358, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 2002, "coverageMaximizingSubsets": 220, "coveredObligations": 3 }, "poolItems": [ { "id": 358, "name": "testing spiritual claims", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "First Thessalonians urges examination before retaining what is good, while First John warns against believing every spirit. Claims of spiritual authority are placed under discernment rather than being accepted solely because such authority is asserted." }, { "id": 1017, "name": "missing proof of life triggers inquiry rather than presumed death", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The draft uses scheduled proof-of-life reporting and priority medical assessment, then initiates a forensic or missing-person process when proof fails or death is alleged. The transition opens an inquiry; it does not turn silence into a verified death or make medical need subordinate to the ordinary sequence." }, { "id": 1570, "name": "true belief held without understanding its grounds", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "Holding an accurate conclusion as unquestioned authority while lacking an understanding of the reasons and objections that would make it a reasoned conviction." }, { "id": 1987, "name": "hypothesis and conclusion of a theorem", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The separation between what a theorem assumes and what it claims follows. Establishing the implication does not establish that the hypothesis holds in a proposed application." }, { "id": 2105, "name": "competing explanations considered as possible joint contributions", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A stance that tests whether proposed mechanisms may both contribute instead of assuming that one must explain every observed case. Compatibility remains a hypothesis requiring discrimination of conditions and effects." }, { "id": 2463, "name": "a public ledger of derivation debts", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A research record naming what a proposed account still has to derive, measure, or explain to support its stronger claims. Publishing the obligations makes progress and omission inspectable while preserving the difference between acknowledging a debt and discharging it." }, { "id": 2525, "name": "verification counters distinguished from evidential strength", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Reported check totals depend on how an implementation groups fields, syndromes and tests. A larger counter is not stronger evidence by itself; comparison identifies which properties were tested, which failures were rejected, and which outputs actually agree." }, { "id": 3248, "name": "inductive reasoning", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 2 ], "definition": "Reasoning from observations or cases toward a broader claim whose truth is not logically guaranteed by those cases alone. Its strength depends on sampling, background assumptions and the alternatives left open." }, { "id": 3379, "name": "convolution", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An operation combining two functions by integrating or summing products as one is reflected and shifted relative to the other. It describes linear shift-invariant responses when those assumptions hold, rather than every possible interaction." }, { "id": 3464, "name": "falsifiability", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "The possibility that a claim could conflict with conceivable observations under specified testing assumptions. Being falsifiable is not the same as being false, and a failed prediction can implicate auxiliary assumptions as well as the focal claim." }, { "id": 4189, "name": "hierarchy of evidence", "grade": 1.5, "judgeA": 2, "judgeB": 1, "agreedObligations": [], "definition": "A heuristic ordering of study designs or evidence types by their expected strength for a specified question, often by susceptibility to bias. The ranking depends on the question, outcome and execution; a high-ranked design is not a guarantee that a particular result is sound." }, { "id": 4201, "name": "criterion-related validity", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "Evidence about how a measure relates to a relevant external outcome or criterion. Concurrent assessment compares at the same time; predictive assessment concerns later outcomes, and evidence for one cannot automatically substitute for the other." }, { "id": 4974, "name": "actor–observer attribution hypothesis", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The proposed tendency for people to explain their own behavior more through situations and others' behavior more through dispositions. Broad evidence has not consistently supported the original general claim, so it should be treated as a hypothesis sensitive to conditions rather than a universal law." }, { "id": 5028, "name": "historical method", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Practices for identifying, evaluating and relating sources to construct accounts of past events and settings. Written, oral and material evidence can contribute, with methods adapted to the question and surviving record rather than one universally sufficient checklist." } ] }, { "task": "B03", "split": "final", "origin": "public-human-question", "domain": "philosophy", "question": "Why is violating the 'only x and y' principle considered a problem for theories of personal identity? Explain how facts constituting identity differ from facts about the naming or history of a person. Use the questioner's example of Bob and his mother to assess whether dependence on a third person actually violates the principle.", "requirements": [ "State what the only x and y principle rules out.", "Distinguish identity conditions from naming or causal history.", "Assess the Bob-and-mother example without assuming its conclusion.", "Explain why a theory's violation of the principle might matter." ], "poolSize": 13, "poolAttainableObligations": [ 1, 2 ], "poolMissingObligations": [ 0, 3 ], "metrics": { "dense": { "top5": [ 3817, 2055, 3869, 4236, 1578 ], "ndcg5": 0.27344128348213886, "agreedObligationCoverage5": 0.25, "coveredObligations": [ 1 ], "agreedAssignments": 1, "labelRedundancy": 0.0, "embeddingRedundancy": 0.3923903016354107, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 3817, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 2055, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3869, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4236, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1578, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3817, 2055, 3869, 4236, 1578 ], "ndcg5": 0.27344128348213886, "agreedObligationCoverage5": 0.25, "coveredObligations": [ 1 ], "agreedAssignments": 1, "labelRedundancy": 0.0, "embeddingRedundancy": 0.3923903016354107, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 3817, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 2055, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3869, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4236, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1578, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 2105, 3817, 1578, 548, 568 ], "ndcg5": 0.11189428907991765, "agreedObligationCoverage5": 0.25, "coveredObligations": [ 1 ], "agreedAssignments": 1, "labelRedundancy": 0.0, "embeddingRedundancy": 0.16922052416143138, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 2105, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3817, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 1578, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 548, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 568, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 4230, 4231, 4235, 3817, 4236 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 1, 2 ], "agreedAssignments": 6, "labelRedundancy": 0.6666666666666666, "embeddingRedundancy": 0.4687949257963882, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 4230, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 1, 2 ] }, { "id": 4231, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] }, { "id": 4235, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 3817, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 4236, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 1287, "coverageMaximizingSubsets": 825, "coveredObligations": 2 }, "poolItems": [ { "id": 548, "name": "Jesus' disavowal of worship directed to himself and Mary", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The Qur'anic judgment dialogue asks Jesus whether he told people to take him and his mother as gods; he denies saying this and states that he instructed worship of his Lord and theirs. His witness is limited to his presence among them, while continuing oversight belongs to God." }, { "id": 568, "name": "selling distinguished from usury", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The Qur’anic distinction between permitted selling and prohibited usury, rejecting the claim that they are simply alike. Renouncing the prohibited gain is joined to retaining the principal and a warning against wronging or being wronged." }, { "id": 1578, "name": "power over another misdescribed as personal liberty", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A claim to freedom in one’s own affairs that actually asserts authority over another person, treating the other’s interests as if they were part of the claimant’s self." }, { "id": 1586, "name": "a personal history reconstructed through successive custodians", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Tracing an unknown past through people who held different parts of its chain of care or transfer, distinguishing each witness’s direct knowledge from what only another can supply." }, { "id": 2010, "name": "a moral example judged by a prior standard", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An exemplary person or act must be assessed by a principle of goodness before serving as a model. Admiration and imitation cannot alone determine the standard that makes the example worthy." }, { "id": 2055, "name": "identity questioned across experienced transformations", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A narrative question in which incompatible experienced standpoints unsettle certainty about which identity supplies the final frame. The question preserves a distinction between experiences while withholding an uncomplicated account of their relation." }, { "id": 2105, "name": "competing explanations considered as possible joint contributions", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A stance that tests whether proposed mechanisms may both contribute instead of assuming that one must explain every observed case. Compatibility remains a hypothesis requiring discrimination of conditions and effects." }, { "id": 3817, "name": "personal identity over time", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "The question of what makes a person at one time the same person at another, despite changes in properties, memory or circumstance. Persistence of a person is distinguished from identifying their characteristics at a single moment." }, { "id": 3869, "name": "social identity theory", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "A framework connecting part of an individual's self-concept to perceived group membership and examining how status, legitimacy, stability and group boundaries shape intergroup behavior. Group identification is distinguished from a complete description of the individual." }, { "id": 4230, "name": "accidental property", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2 ], "definition": "A property an entity can lack or change while remaining the same entity under the relevant account of identity. Accidental here contrasts with essential, not with deliberately chosen or statistically unlikely; a property can be important without being identity-constituting." }, { "id": 4231, "name": "essence as identity-constituting properties", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2 ], "definition": "The property or set of properties taken to make an entity what it is, such that losing them would lose its identity. This contrasts with properties it can possess contingently; identifying an essence is a philosophical commitment rather than simply choosing a useful classification." }, { "id": 4235, "name": "numerical identity", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "The relation of being one and the same entity, distinguished from sharing qualities or being alike under a criterion. Two identical-looking records can remain two tokens; deciding persistence through change requires an account of the relevant entity." }, { "id": 4236, "name": "identity of indiscernibles", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "The principle that entities sharing every relevant property are one and the same entity. Its content depends on which properties count; it is distinct from the converse principle that one entity cannot differ from itself and from empirical similarity under a limited test." } ] }, { "task": "B04", "split": "final", "origin": "public-human-question", "domain": "philosophy", "question": "What theoretical and practical value does philosophical work on personal identity have? The questioner doubts that it helps identify someone in an old photograph or solve a police identification problem. Distinguish deciding what makes someone the same person from obtaining evidence about who someone is, and explain why a person might care about either issue.", "requirements": [ "Distinguish persistence or identity criteria from evidential identification.", "Address the photograph and police examples as objections.", "Give a defensible account of theoretical value.", "Explain a practical reason to care while acknowledging limits." ], "poolSize": 17, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3817, 2055, 3869, 5040, 3863 ], "ndcg5": 0.6995577437011554, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.2, "embeddingRedundancy": 0.38125703148555845, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3817, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 1, 2 ] }, { "id": 2055, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3869, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5040, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3863, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3817, 2055, 3869, 5040, 3863 ], "ndcg5": 0.6995577437011554, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.2, "embeddingRedundancy": 0.38125703148555845, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3817, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 1, 2 ] }, { "id": 2055, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3869, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5040, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3863, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3817, 1334, 4235, 3853, 4594 ], "ndcg5": 0.8079283296341391, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.5, "embeddingRedundancy": 0.15739007504659314, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3817, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 1, 2 ] }, { "id": 1334, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4235, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [] }, { "id": 3853, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 4594, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3817, 4235, 2055, 3853, 1402 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.6, "embeddingRedundancy": 0.3434285960861677, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 3817, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 1, 2 ] }, { "id": 4235, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [] }, { "id": 2055, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3853, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] }, { "id": 1402, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 6188, "coverageMaximizingSubsets": 1469, "coveredObligations": 4 }, "poolItems": [ { "id": 1334, "name": "confidentiality within an entrusted care relationship", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Restraint in sharing personal information learned through care, distinguishing disclosure to someone entitled to ask from gossip or answering a questioner without that entitlement." }, { "id": 1402, "name": "another person as a centre of experience", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The recognition that another person’s needs, memories and perspective organize a lived world that is not merely a role in one’s own story. Imagining service to that person is distinguished from apprehending how the same situation may feel from their position." }, { "id": 1691, "name": "identification with a stranger through an incomplete report", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A response in which another person’s briefly reported fate becomes a way to reconsider one’s own life, while the imagined connection exceeds what is actually known about that person." }, { "id": 1932, "name": "intimate and professional authority combined", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An arrangement in which the same person holds intimate and professional authority over someone, so questioning a practical judgment can also be treated as disloyalty within the relationship. The two roles can make an appeal difficult even when care is sincerely professed." }, { "id": 2055, "name": "identity questioned across experienced transformations", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "A narrative question in which incompatible experienced standpoints unsettle certainty about which identity supplies the final frame. The question preserves a distinction between experiences while withholding an uncomplicated account of their relation." }, { "id": 2078, "name": "empathetic identification through an adopted first person", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "Speaking as if one becomes another sufferer rather than merely observing that person. The device can intensify solidarity while leaving a boundary between imaginative identification and the other person’s independently known experience." }, { "id": 2689, "name": "computational reduction", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A specified transformation or procedure that uses a solution to one problem to solve another. Its direction and allowed computational resources determine what difficulty claim follows; reducing A to B makes B a resource for A, not a proof that A and B are identical." }, { "id": 3622, "name": "electron configuration", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The distribution of electrons among an atom's or molecule's quantum states in a specified description. It helps explain bonding and periodic behavior; an orbital diagram is a model, not a photograph of electron paths." }, { "id": 3817, "name": "personal identity over time", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2 ], "definition": "The question of what makes a person at one time the same person at another, despite changes in properties, memory or circumstance. Persistence of a person is distinguished from identifying their characteristics at a single moment." }, { "id": 3853, "name": "philosophical pragmatism", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "A philosophical tradition that investigates concepts, beliefs and inquiry through their practical bearings on experience and action. Attention to consequences helps clarify meaning and warrant without making whatever is immediately convenient automatically true or right." }, { "id": 3863, "name": "philosophical humanism", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A family of outlooks placing human agency, welfare, dignity and the development of human capacities at the center of ethical or philosophical concern. Its historical varieties differ; concern for humans does not by definition settle every obligation to other beings." }, { "id": 3869, "name": "social identity theory", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A framework connecting part of an individual's self-concept to perceived group membership and examining how status, legitimacy, stability and group boundaries shape intergroup behavior. Group identification is distinguished from a complete description of the individual." }, { "id": 4235, "name": "numerical identity", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2 ], "definition": "The relation of being one and the same entity, distinguished from sharing qualities or being alike under a criterion. Two identical-looking records can remain two tokens; deciding persistence through change requires an account of the relevant entity." }, { "id": 4594, "name": "maximum a posteriori estimation", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Selecting a parameter value that maximizes its posterior density under a specified parameterization. It combines likelihood and prior into a point estimate rather than retaining the full posterior." }, { "id": 4926, "name": "problem-focused coping", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Coping directed toward changing a source of stress or the practical conditions producing it, such as obtaining information or developing needed skills. Its suitability depends partly on what can actually be influenced; inability to change a stressor is not evidence of personal failure." }, { "id": 4991, "name": "identification in social influence", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Changing attitudes or behavior in relation to someone admired or to a desired relationship with them. The attraction of the person or relationship helps explain the influence, separately from the independent merits of the adopted position." }, { "id": 5040, "name": "cultural identity", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A person's or group's sense of connection with cultural practices, histories and affiliations. It can be multiple, negotiated and changing, rather than fixed solely by ancestry or a label assigned by others." } ] }, { "task": "B05", "split": "final", "origin": "public-human-question", "domain": "biology", "question": "Do introduced invasive species usually cause lasting ecological damage, or can ecosystems adjust over time? Distinguish a decline in the invader's abundance from recovery of native species and ecosystem function. Explain how predator responses, resource depletion, and competition could produce different long-term outcomes, and whether outcomes differ across broad groups of invaders.", "requirements": [ "Distinguish invader abundance, native-species recovery, and ecosystem-level change.", "Compare at least two mechanisms that could reduce an invader's population.", "Address differences across plants, arthropods, or mammals without assuming one universal pattern.", "Explain what observations would support a claim of long-term adjustment." ], "poolSize": 16, "poolAttainableObligations": [ 0, 1, 3 ], "poolMissingObligations": [ 2 ], "metrics": { "dense": { "top5": [ 3106, 4341, 3125, 3103, 1697 ], "ndcg5": 0.6534046191206433, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.7142857142857143, "embeddingRedundancy": 0.46304445142872713, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 3106, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 4341, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3125, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 3103, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 1697, "agreedObligations": [ 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3106, 4341, 3125, 3103, 1697 ], "ndcg5": 0.6534046191206433, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.7142857142857143, "embeddingRedundancy": 0.46304445142872713, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 3106, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 4341, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3125, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 3103, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 1697, "agreedObligations": [ 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 1697, 3098, 3106, 1887, 3086 ], "ndcg5": 0.5665406937160657, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.5, "embeddingRedundancy": 0.31171862087460755, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 1697, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3098, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 3106, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0 ] }, { "id": 1887, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3086, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 1 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 }, "oracle": { "top5": [ 3098, 3103, 3124, 3125, 5767 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 9, "labelRedundancy": 0.6666666666666666, "embeddingRedundancy": 0.5080797253791831, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3098, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 3103, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 3124, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 3125, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 5767, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 1 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 4368, "coverageMaximizingSubsets": 2332, "coveredObligations": 3 }, "poolItems": [ { "id": 1697, "name": "preserving ecological relations before a habitat is transformed", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "Recording activities, associations and environmental conditions while they can still be observed, because specimens alone cannot later reconstruct relationships erased by transformation." }, { "id": 1887, "name": "small initial uncertainty amplified into divergent outcomes", "grade": 1.5, "judgeA": 2, "judgeB": 1, "agreedObligations": [], "definition": "In a sensitive process, slight differences in initial conditions can grow into large differences in later outcomes. Knowing a governing law therefore does not by itself make a prediction precise when the initial state is known only approximately." }, { "id": 2346, "name": "recovery and spread distinguished from immediate response", "grade": 2.5, "judgeA": 3, "judgeB": 2, "agreedObligations": [ 0, 3 ], "definition": "Two interventions can produce a similar immediate visible reaction yet differ in how far the effect spreads, whether it reverses and what later damage follows. Following those separate dimensions can distinguish responses that a single endpoint would group together." }, { "id": 3086, "name": "biodiversity", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 3 ], "definition": "Variation in living systems considered across such levels as genes, organisms, species and ecosystems. Different measures capture different parts of this variation; one species count cannot stand for every aspect of biological diversity." }, { "id": 3098, "name": "ecological competition", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "An interaction in which organisms' use of a limiting resource reduces its availability to one another, affecting performance or reproduction. This resource-based sense is distinct from apparent competition mediated through a shared enemy." }, { "id": 3103, "name": "ecological succession", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "Change in the composition and structure of an ecological community through time, including colonization of new habitat and change following disturbance. Patterns can recur without requiring every site to follow one inevitable sequence toward a fixed final community." }, { "id": 3106, "name": "invasive species", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "An introduced species that establishes and spreads with harmful effects in a receiving environment. Being non-native alone does not establish invasiveness; ecological setting, spread and the nature of harm must be considered." }, { "id": 3109, "name": "conservation biology", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An interdisciplinary field concerned with understanding and limiting biodiversity loss, combining ecological and evolutionary evidence with the social conditions of protection and resource use. Its goals involve values as well as empirical claims." }, { "id": 3110, "name": "ecological restoration", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 3 ], "definition": "Assisting the recovery of an ecosystem that has been damaged, degraded or destroyed through interventions guided by ecological understanding and explicit goals. Recovery need not mean exact reproduction of one historical condition." }, { "id": 3124, "name": "ecological resistance", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "The degree to which an ecosystem's specified properties change little when a disturbance occurs. It concerns the immediate response relative to a chosen reference, and differs from the capacity to reorganize or recover while retaining important functions." }, { "id": 3125, "name": "ecological resilience", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "An ecosystem's capacity to absorb disturbance and reorganize while retaining characteristic functions, structure and feedbacks. Resilience need not mean returning to an identical prior state, and its assessment depends on the functions and boundaries chosen." }, { "id": 3202, "name": "closed ecological system", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An ecosystem intended to sustain life without ongoing matter exchange across its declared boundary, while receiving energy from outside. Internal biological and physical processes recycle materials; closure of matter does not imply isolation from energy or perfect long-term stability." }, { "id": 4341, "name": "ecology", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "The study of interactions among organisms and their environments, including abundance, distribution, energy and material flows across populations, communities and ecosystems. Ecological description does not by itself determine what society ought to protect." }, { "id": 5767, "name": "extinction debt", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "Expected future losses of species resulting from past ecological disruption, delayed because populations can persist for a time after conditions for long-term replacement have deteriorated." }, { "id": 5782, "name": "apparent competition", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "An indirect negative interaction in which one prey or host population increases pressure from a shared enemy on another, without requiring competition for the same resource." }, { "id": 5785, "name": "ecological functional redundancy", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0 ], "definition": "Overlap in the contribution of different species to a specified ecosystem function, despite differences in taxonomy or other traits." } ] }, { "task": "B06", "split": "final", "origin": "public-human-question", "domain": "biology", "question": "A field managed by sheep grazing has higher measured floral biodiversity than a field managed by mowing. Does adaptation to grazing, together with nutrients returned by animals, adequately explain this contrast? Compare these explanations with other mechanisms and with differences between the fields, and identify evidence needed before attributing the difference to management.", "requirements": [ "Assess the proposed adaptation and nutrient-return explanations.", "Compare at least one other plausible ecological mechanism.", "Distinguish observed association from a management effect.", "Identify additional measurements or a comparison design that could separate explanations." ], "poolSize": 18, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 771, 4413, 5845, 5824, 1951 ], "ndcg5": 0.5114237404054409, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.4, "embeddingRedundancy": 0.3070042663576523, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 771, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4413, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 5845, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 5824, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] }, { "id": 1951, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 771, 4413, 5845, 5824, 1951 ], "ndcg5": 0.5114237404054409, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.4, "embeddingRedundancy": 0.3070042663576523, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 771, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4413, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 5845, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 5824, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] }, { "id": 1951, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 2105, 4413, 771, 4456, 3467 ], "ndcg5": 0.7157882110875621, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.5, "embeddingRedundancy": 0.09743147892016321, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 2105, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [ 0, 2, 3 ] }, { "id": 4413, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 771, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4456, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3467, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [ 1 ], "uniqueObligationsGainedVsDense": [ 2 ], "lostCountVsDense": 1, "gainedCountVsDense": 1 }, "oracle": { "top5": [ 2105, 3467, 4413, 5824, 1951 ], "ndcg5": 0.9675180423472418, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 9, "labelRedundancy": 0.5555555555555556, "embeddingRedundancy": 0.21896478758692228, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 2105, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [ 0, 2, 3 ] }, { "id": 3467, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] }, { "id": 4413, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 5824, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 1951, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 8568, "coverageMaximizingSubsets": 1328, "coveredObligations": 4 }, "poolItems": [ { "id": 592, "name": "peoples and tribes for mutual recognition", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Human difference into peoples and tribes is connected to knowing one another, while honour before God is measured by fear of God. Shared origin and plurality remain together without turning social descent into the stated measure of worth." }, { "id": 771, "name": "fallow land as provision for poor people and animals", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The seventh-year instruction leaves cultivated land untended so poor people may eat and wild animals may take what remains. Agricultural cessation becomes an opening of access to produce, extending the consequence of rest beyond the landholder’s own household." }, { "id": 1276, "name": "reciprocal adaptation between interacting lineages", "grade": 2.5, "judgeA": 3, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "A proposed process in which advantageous changes within one lineage alter the conditions faced by another, allowing successive changes in their interaction without presuming a common intention or perfectly shared interests." }, { "id": 1670, "name": "attainment hidden because disclosure creates danger", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A measurement problem in which people conceal an acquired ability because revealing it could bring punishment, removal or persecution, making recorded attainment an uncertain guide to actual ability." }, { "id": 1951, "name": "pollination through animal foraging", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 3 ], "definition": "A relation in which an animal’s search for food transfers pollen among flowers. Benefit to reproduction and benefit to the visitor can be connected without implying that either organism consciously intends the other’s benefit." }, { "id": 2099, "name": "an analogy tested through a new predicted difference", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "Using a resemblance between systems to predict an unmeasured contrast, then checking that contrast where conditions change. Agreement on the first resemblance is a reason to investigate further, not proof of complete equivalence." }, { "id": 2105, "name": "competing explanations considered as possible joint contributions", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 2, 3 ], "definition": "A stance that tests whether proposed mechanisms may both contribute instead of assuming that one must explain every observed case. Compatibility remains a hypothesis requiring discrimination of conditions and effects." }, { "id": 3086, "name": "biodiversity", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "Variation in living systems considered across such levels as genes, organisms, species and ecosystems. Different measures capture different parts of this variation; one species count cannot stand for every aspect of biological diversity." }, { "id": 3109, "name": "conservation biology", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An interdisciplinary field concerned with understanding and limiting biodiversity loss, combining ecological and evolutionary evidence with the social conditions of protection and resource use. Its goals involve values as well as empirical claims." }, { "id": 3467, "name": "internal validity", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "The credibility of a causal conclusion within a study's design and setting, including whether alternative explanations for the observed effect have been adequately addressed. It does not itself establish transfer to other settings." }, { "id": 4067, "name": "visual contrast", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A difference in luminance or color relative to a background or neighboring region that makes features distinguishable. Perceived contrast depends on the surrounding pattern and viewing conditions as well as the stored or measured values." }, { "id": 4413, "name": "ecological functional diversity", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "Variation in traits or ecological roles among organisms in a community, such as feeding strategies or resource-use patterns. It can differ from species count; how traits are selected and measured affects the diversity represented." }, { "id": 4456, "name": "finite field", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A field with finitely many elements, necessarily a prime-power number. Fields of a given allowed size are isomorphic, but an explicit representation is still needed to calculate with encoded elements." }, { "id": 5108, "name": "stock–flow distinction in economic accounts", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The distinction between a quantity measured at a point in time, such as assets or debt, and activity measured over a period, such as income or expenditure. Flows help explain changes in stocks, but revaluation and other adjustments may also be needed." }, { "id": 5824, "name": "plant defense against herbivory", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0 ], "definition": "Plant traits and responses that reduce the impact of consumers through deterrence, reduced digestibility, avoidance, recovery or recruitment of herbivore enemies." }, { "id": 5844, "name": "agroforestry", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Intentional integration of trees or other woody perennials with crops or pasture in a managed land-use system." }, { "id": 5845, "name": "conservation agriculture", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "A farming approach organized around limited soil disturbance, sustained soil cover and diversification of cultivated species, aiming to maintain production while protecting soil processes." }, { "id": 5846, "name": "crop diversity", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Variation among and within cultivated crop species, including genetic and phenotypic differences relevant to production, adaptation and future breeding." } ] }, { "task": "B07", "split": "development", "origin": "public-human-question", "domain": "biology", "question": "When should a population biologist use the Moran model rather than the Wright-Fisher model of genetic drift? Compare their assumptions about population size and reproduction, the biological populations each can represent, and the empirical observations that would help choose between them. Explain model limitations rather than only listing equations.", "requirements": [ "Compare demographic and reproductive assumptions of the two models.", "Relate assumptions to candidate biological populations.", "Explain a meaningful advantage or limitation of each model.", "Describe observations or experiments that could discriminate their fit." ], "poolSize": 11, "poolAttainableObligations": [ 0, 1, 2 ], "poolMissingObligations": [ 3 ], "metrics": { "dense": { "top5": [ 5753, 3127, 3128, 5768, 3131 ], "ndcg5": 0.8347099844293421, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 2 ], "agreedAssignments": 6, "labelRedundancy": 0.5, "embeddingRedundancy": 0.5309786131630738, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 5753, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 1, 2 ] }, { "id": 3127, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3128, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 5768, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3131, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 5753, 3127, 3128, 5768, 3131 ], "ndcg5": 0.8347099844293421, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 2 ], "agreedAssignments": 6, "labelRedundancy": 0.5, "embeddingRedundancy": 0.5309786131630738, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 5753, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 1, 2 ] }, { "id": 3127, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3128, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 5768, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3131, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 5753, 3100, 3130, 1881, 3442 ], "ndcg5": 0.8049969536740027, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 2 ], "agreedAssignments": 5, "labelRedundancy": 0.4, "embeddingRedundancy": 0.25196504407431564, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 5753, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 1, 2 ] }, { "id": 3100, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 3130, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1881, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3442, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 5753, 3100, 3128, 3129, 3131 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 2 ], "agreedAssignments": 10, "labelRedundancy": 0.7, "embeddingRedundancy": 0.5037693964213433, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 5753, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 1, 2 ] }, { "id": 3100, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 3128, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 3129, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] }, { "id": 3131, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 462, "coverageMaximizingSubsets": 386, "coveredObligations": 3 }, "poolItems": [ { "id": 1881, "name": "stages between a felt event and a theoretical explanation", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "Poincaré separates an impression, its verbal report, an instrument reading, a corrected reading and a theoretical interpretation. Calling them all a fact can hide the different transformations and warrants involved." }, { "id": 3100, "name": "population dynamics", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 1 ], "definition": "Changes in the size and composition of a population through time, studied through processes and models of birth, death, movement and interaction. A model specifies which processes and environmental conditions it includes." }, { "id": 3127, "name": "biological evolution", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Change in heritable characteristics of populations across generations through processes including mutation, selection, drift and gene flow. It concerns population history rather than an individual's lifetime development and does not imply a predetermined direction toward greater worth or complexity." }, { "id": 3128, "name": "genetic drift", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "Change in allele frequencies through chance differences in transmission across generations in a finite population. Drift can alter or eliminate variants without those changes being explained by a selective advantage." }, { "id": 3129, "name": "gene flow", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 2 ], "definition": "Transfer of genetic variants between populations through successful movement and reproduction or other genetic exchange. It changes genetic composition and can oppose or contribute to differentiation; physical arrival alone does not establish gene flow." }, { "id": 3130, "name": "genetic variation", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Differences in genetic sequences or variants among individuals or populations. Their distribution can change through mutation, recombination, migration, selection and drift; genetic variation is distinct from all observable phenotypic variation." }, { "id": 3131, "name": "population genetics", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 2 ], "definition": "Study of how genetic variants are distributed and change within and among populations, using models and observations of processes such as selection, drift, mutation and migration. Its conclusions depend on population structure, sampling and the assumptions of the model used." }, { "id": 3442, "name": "bootstrap resampling", "grade": 0.5, "judgeA": 0, "judgeB": 1, "agreedObligations": [], "definition": "Estimating a statistic's variability by repeatedly resampling from an empirical sample under a specified scheme. The procedure inherits limitations of the sample and resampling model; it does not create missing population information." }, { "id": 5753, "name": "effective population size", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2 ], "definition": "The size of an idealized population that matches a specified genetic property, such as drift or inbreeding, of a real population under stated model assumptions." }, { "id": 5756, "name": "population bottleneck", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 2 ], "definition": "A sharp reduction in population size that can alter allele frequencies through sampling and drift, remove genetic variation and increase relatedness among survivors." }, { "id": 5768, "name": "population viability analysis", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A species- or population-specific assessment estimating persistence or extinction risk over a stated period under demographic, environmental and management assumptions." } ] }, { "task": "B08", "split": "final", "origin": "public-human-question", "domain": "biology", "question": "When, if ever, should a species introduced to a region cease being classified as invasive or non-native? Compare human introduction with natural range expansion, and distinguish time since arrival from ecological effects or equilibrium. Explain which criteria justify changing a classification when a once disruptive species becomes integrated into a community.", "requirements": [ "Distinguish origin-based labels from labels based on ecological impact.", "Compare natural colonization with human-mediated introduction.", "Evaluate elapsed time and ecological equilibrium as classification criteria.", "Explain what evidence and conventions would be needed to change a label." ], "poolSize": 12, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3106, 1825, 5766, 1281, 5771 ], "ndcg5": 0.7982296316051806, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.3971610359112747, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3106, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 1825, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 5766, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1281, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5771, "agreedObligations": [ 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3106, 1825, 5766, 1281, 5771 ], "ndcg5": 0.7982296316051806, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.3971610359112747, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3106, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 1825, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 5766, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1281, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5771, "agreedObligations": [ 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3103, 1825, 3106, 79, 2387 ], "ndcg5": 0.8587958726494572, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.0, "embeddingRedundancy": 0.2068993523759152, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3103, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 1825, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 3106, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 79, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2387, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 }, "oracle": { "top5": [ 1825, 3103, 3106, 1693, 5771 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.42857142857142855, "embeddingRedundancy": 0.36575923704384805, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 1825, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 3103, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 3106, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 1693, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] }, { "id": 5771, "agreedObligations": [ 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 792, "coverageMaximizingSubsets": 113, "coveredObligations": 4 }, "poolItems": [ { "id": 79, "name": "non-invasive neural bridge", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An interface that translates measurable neural activity into controllable signals without penetrating neural tissue, with reversibility, consent, and understandable control treated as design requirements." }, { "id": 1281, "name": "local disappearance distinguished from extinction", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "The distinction between absence from one place or sampled interval and the ending of a lineage everywhere, allowing migration or continued survival elsewhere to explain some apparent disappearances and returns." }, { "id": 1693, "name": "a cohabiting ecological community as a unit of inquiry", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "Organisms sharing a habitat considered together through their interrelations, responses and effects on conditions, rather than merely as an inventory of independently described species." }, { "id": 1825, "name": "unintended biological introduction along human routes", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "Organisms can enter new regions through transport materials, canals or movements intended for other purposes. Human introduction is not restricted to species deliberately brought for cultivation or use." }, { "id": 2387, "name": "central action that increases local governing capacity", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Shared institutions can provide resources and coordination while leaving local participants responsible for identifying needs, administering action and learning from experience. The democratic effect depends on how central action supports local initiative, not simply on whether authority is labeled central or local." }, { "id": 3103, "name": "ecological succession", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2 ], "definition": "Change in the composition and structure of an ecological community through time, including colonization of new habitat and change following disturbance. Patterns can recur without requiring every site to follow one inevitable sequence toward a fixed final community." }, { "id": 3104, "name": "keystone species", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "A species whose effect on a community is disproportionately large relative to its abundance. The designation is a claim about ecological influence in a particular system, not simply a label for a large, rare or culturally important species." }, { "id": 3106, "name": "invasive species", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "An introduced species that establishes and spreads with harmful effects in a receiving environment. Being non-native alone does not establish invasiveness; ecological setting, spread and the nature of harm must be considered." }, { "id": 3137, "name": "speciation", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Evolutionary formation of distinct species through divergence of populations, involving processes that can include reproductive isolation and ecological differentiation. Its identification depends partly on the species concept and evidence used; it need not be one sudden universal threshold." }, { "id": 5498, "name": "repeat-expansion disorder", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A genetic disorder associated with expansion of a repeated DNA sequence beyond a context-dependent range, potentially disrupting gene regulation or producing harmful RNA or protein effects." }, { "id": 5766, "name": "species concept", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A criterion or framework for delimiting species, such as reproductive compatibility, evolutionary lineage, morphology or ecological differentiation, with different applicability across organisms and evidence." }, { "id": 5771, "name": "metapopulation", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "A set of spatially separated populations of the same species linked by movement, with regional persistence shaped by local extinction, colonization and exchange." } ] }, { "task": "B09", "split": "development", "origin": "public-human-question", "domain": "economics", "question": "If the aim is to raise low-paid workers' living standards, what advantages could a minimum wage have over a wage subsidy? Compare the predicted employment effects of a binding wage floor with the financing and distribution of a subsidy. Explain the conditions under which one instrument would be preferable, separating economic mechanisms from political popularity.", "requirements": [ "Compare how each instrument changes employer and worker incentives.", "Discuss employment and income effects under stated market assumptions.", "Identify who finances and who benefits from each approach.", "Give conditional reasons to prefer either instrument." ], "poolSize": 14, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 5055, 5053, 5052, 5054, 5091 ], "ndcg5": 0.7991766257901849, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.7, "embeddingRedundancy": 0.39893570502201064, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 5055, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 5053, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 5052, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5054, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [] }, { "id": 5091, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 5055, 5053, 5052, 5054, 5091 ], "ndcg5": 0.7991766257901849, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.7, "embeddingRedundancy": 0.39893570502201064, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 5055, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 5053, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 5052, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5054, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [] }, { "id": 5091, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 4352, 5055, 2105, 5624, 5116 ], "ndcg5": 0.34639919963866117, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.0, "embeddingRedundancy": 0.14725434947326094, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 4352, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5055, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 2105, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5624, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5116, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 }, "oracle": { "top5": [ 5091, 5054, 5055, 5053, 5116 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 11, "labelRedundancy": 0.6363636363636364, "embeddingRedundancy": 0.27937374026267736, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 5091, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 5054, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 5055, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [] }, { "id": 5053, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 5116, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 2002, "coverageMaximizingSubsets": 385, "coveredObligations": 4 }, "poolItems": [ { "id": 785, "name": "a poor worker’s wage is not deferred overnight", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Deuteronomy requires payment of a poor hired worker on the day of labor, before sunset, because the worker sets the heart upon it. Economic dependency makes delay itself consequential, even where the employer has not denied that payment is owed." }, { "id": 988, "name": "temporary transport-risk support linked to verified calm", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A draft financing device offsetting part of disruption-related freight or insurance costs while a route stabilizes, then withdrawing the subsidy as verified incident rates improve. It distinguishes transitional support from a permanent subsidy or proof that every participating route is safe." }, { "id": 2105, "name": "competing explanations considered as possible joint contributions", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A stance that tests whether proposed mechanisms may both contribute instead of assuming that one must explain every observed case. Compatibility remains a hypothesis requiring discrimination of conditions and effects." }, { "id": 4352, "name": "labour economics", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The study of labour as an input to production and of workers, employers and institutions in labour markets. Wages, participation, bargaining and discrimination are different questions; describing a market does not establish that its outcomes are fair." }, { "id": 5052, "name": "wage", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Remuneration for labor, commonly specified per unit of time or work and sometimes used more broadly for employee pay. Wage rates are shaped by bargaining, institutions and markets; payment level is not a complete measure of work's social value." }, { "id": 5053, "name": "reservation wage", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 1 ], "definition": "The lowest wage at which a person would accept a particular job under specified conditions and available alternatives. It may change with household needs, risks and job characteristics, so it is not simply a fixed measure of willingness to work." }, { "id": 5054, "name": "efficiency wage", "grade": 2.5, "judgeA": 3, "judgeB": 2, "agreedObligations": [ 0, 1, 3 ], "definition": "A wage above a modeled market-clearing level that an employer may pay to improve performance, recruitment or retention. Efficiency-wage theories explain possible incentives for this choice without implying that higher pay always produces a proportional productivity increase." }, { "id": 5055, "name": "minimum wage", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 0, 1, 3 ], "definition": "A legally established lower bound on remuneration for covered employment. Coverage, rates and enforcement vary by jurisdiction and worker category; its economic effects depend on market conditions and policy design." }, { "id": 5057, "name": "collective labor agreement", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An agreement resulting from negotiations between workers' representatives and employers, setting specified employment terms or procedures. Its scope, binding effect and implementation depend on the agreement and relevant legal framework." }, { "id": 5058, "name": "trade union", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An organization of workers formed to represent shared employment interests and improve or protect working conditions. It can bargain collectively and provide representation, while its internal governance and membership scope vary." }, { "id": 5091, "name": "price floor", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "A lower limit imposed on the price of a specified good or service. Whether it binds depends on transaction conditions and the relevant market; in a simple competitive model a floor above equilibrium changes quantities supplied and demanded." }, { "id": 5116, "name": "fiscal policy", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "Use of public spending, taxation and related budget decisions to influence economic activity, distribution or resource allocation. It differs from monetary policy even when both interact, and its effects depend on timing, institutions and economic conditions." }, { "id": 5117, "name": "monetary policy", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Actions by a monetary authority to influence monetary and financial conditions in pursuit of stated objectives, using instruments such as interest rates, asset operations or communication. Tools and mandates vary, and effects operate through uncertain transmission channels." }, { "id": 5624, "name": "ventilation–perfusion ratio", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The ratio of air flow reaching an alveolar region to blood flow through its exchange vessels, describing how respiratory ventilation and perfusion are matched." } ] }, { "task": "B10", "split": "final", "origin": "public-human-question", "domain": "economics", "question": "Under what conditions could price controls increase competition between firms? Distinguish lower regulated prices from stronger competitive pressure, and explain how effects on entry, exit, quality, or other forms of rivalry might differ across market structures. Identify a defensible mechanism and the evidence needed to show that competition, rather than only prices, changed.", "requirements": [ "Define the competition outcome being claimed.", "Provide or assess a mechanism linking a specified control to rivalry.", "Explain a countervailing effect on firms or consumers.", "Identify evidence that distinguishes competition from a simple price change." ], "poolSize": 13, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 5065, 1608, 5062, 5067, 1482 ], "ndcg5": 0.680725678280414, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.5, "embeddingRedundancy": 0.4149722713528776, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 5065, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 1608, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 5062, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 5067, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 1482, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 5065, 1608, 5062, 5067, 1482 ], "ndcg5": 0.680725678280414, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.5, "embeddingRedundancy": 0.4149722713528776, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 5065, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 1608, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 5062, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 5067, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 1482, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 5063, 3098, 1873, 1608, 5141 ], "ndcg5": 0.45781220159613184, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.4, "embeddingRedundancy": 0.24481008598979054, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 5063, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 3098, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1873, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 1608, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 5141, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [ 2 ], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 1, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 1482, 1873, 5062, 5073, 5091 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 11, "labelRedundancy": 0.6363636363636364, "embeddingRedundancy": 0.3712091365670169, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 1482, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 1873, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0 ] }, { "id": 5062, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 5073, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [] }, { "id": 5091, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 1287, "coverageMaximizingSubsets": 880, "coveredObligations": 4 }, "poolItems": [ { "id": 1482, "name": "competition constrained by unverifiable quality", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "A market condition in which buyers cannot readily test what is offered, allowing inferior goods to appear competitive and making reliable judgment part of the cost of exchange." }, { "id": 1608, "name": "a price change with alternative value movements", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "The distinction between a commodity becoming more costly relative to other goods and the money used to price it becoming less valuable, either of which can raise its quoted money price." }, { "id": 1873, "name": "production economies distinguished from consumer benefit", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "A business can reduce its costs without automatically passing the saving to buyers. Explaining lower prices requires examining how savings, competitive pressure and pricing decisions interact, rather than treating consolidation alone as the cause." }, { "id": 3098, "name": "ecological competition", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An interaction in which organisms' use of a limiting resource reduces its availability to one another, affecting performance or reproduction. This resource-based sense is distinct from apparent competition mediated through a shared enemy." }, { "id": 3928, "name": "supply-and-demand model", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 2 ], "definition": "A model relating quantities sellers offer and buyers seek to price and other specified conditions. In the competitive market-clearing version, equilibrium equates supplied and demanded quantities; changes in conditions shift the schedules rather than merely moving along them." }, { "id": 5059, "name": "monopsony", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "A market structure with a single buyer facing multiple potential sellers. Related monopsony power can arise when buyers face limited competition, allowing purchase terms to differ from competitive conditions; the relevant market must be specified." }, { "id": 5062, "name": "oligopoly", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "A market structure dominated by a small number of suppliers whose decisions significantly affect one another. Strategic interdependence can support competition or coordination; few firms do not by themselves establish collusion." }, { "id": 5063, "name": "perfect competition", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "An idealized market model in which individual participants cannot influence price, with assumptions such as many buyers and sellers, substitutable products and sufficiently open entry. Results about efficiency depend on the full model assumptions rather than competition as a slogan." }, { "id": 5065, "name": "monopolistic competition", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "A market model with many sellers offering differentiated products, giving each some influence over its own price while facing substitutes and relatively open entry. It combines product differentiation with competition rather than being a synonym for monopoly." }, { "id": 5067, "name": "price discrimination", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "Charging different buyers different prices for comparable goods or services in ways not fully explained by supply-cost differences. The strategy uses variation in willingness to pay and requires some control over pricing and resale or access between segments." }, { "id": 5073, "name": "market contestability", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 3 ], "definition": "The extent to which feasible entry and exit by potential competitors constrain established firms. A market with few current suppliers can face competitive pressure if entry is credible; the perfect model assumes very low barriers, no sunk costs and comparable technology." }, { "id": 5091, "name": "price floor", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2 ], "definition": "A lower limit imposed on the price of a specified good or service. Whether it binds depends on transaction conditions and the relevant market; in a simple competitive model a floor above equilibrium changes quantities supplied and demanded." }, { "id": 5141, "name": "heritage language", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A language connected to a person's family or community background, often differing from the surrounding dominant language. Uses of the term vary: some require childhood exposure or proficiency, while others include cultural affiliation without current fluency." } ] }, { "task": "B11", "split": "final", "origin": "public-human-question", "domain": "economics", "question": "How can public roads be called public goods when only a limited number of people can use them at once? Distinguish public ownership from non-rivalry and exclusion, and explain how congestion changes the classification. Compare a lightly used road with the same road at capacity without treating physical capacity alone as a complete answer.", "requirements": [ "Distinguish government provision from the economic properties of a good.", "Explain rivalry and exclusion as separate dimensions.", "Compare use below capacity with congestion at capacity.", "Clarify the limits of calling every public road a pure public good." ], "poolSize": 13, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3874, 5101, 2204, 4318, 5264 ], "ndcg5": 0.6303994075567672, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.4, "embeddingRedundancy": 0.43780582953701297, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3874, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 5101, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 2204, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4318, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5264, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3874, 5101, 2204, 4318, 5264 ], "ndcg5": 0.6303994075567672, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.4, "embeddingRedundancy": 0.43780582953701297, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3874, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 5101, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 2204, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4318, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5264, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3874, 2048, 3876, 5975, 4163 ], "ndcg5": 0.4876689501397588, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.17163795875343402, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3874, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 2048, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3876, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 5975, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4163, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3874, 5099, 5101, 2138, 5100 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 12, "labelRedundancy": 0.6666666666666666, "embeddingRedundancy": 0.4467248782398235, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3874, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 5099, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [] }, { "id": 5101, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 2138, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2 ] }, { "id": 5100, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 1287, "coverageMaximizingSubsets": 390, "coveredObligations": 4 }, "poolItems": [ { "id": 1674, "name": "authorized public funding blocked by officeholder exclusion", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A public benefit withheld from an eligible group on the ground that the people it has selected to administer the benefit are ineligible to hold the required office." }, { "id": 2048, "name": "capacity distinguished from its application", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The same effective capacity can yield different outcomes when placed in different activities, scales or institutions. Assessing a thing only through its customary use can conceal another application without establishing that every new use is desirable." }, { "id": 2138, "name": "economic classification dependent on an asset’s use", "grade": 2.5, "judgeA": 3, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "Classification of an item according to the role it performs in production, exchange or consumption, rather than solely its physical kind. The same tool can occupy different categories under different uses, provided the chosen definitions remain explicit." }, { "id": 2204, "name": "public use as a means of developing knowledge", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Making knowledge intelligible and available to many people exposes it to correction, new consequences and adaptation to varied purposes. Wider circulation can improve its content as well as increase the number who possess it." }, { "id": 3874, "name": "public good in economics", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 3 ], "definition": "A good whose use is non-rival and from which excluding users is difficult or infeasible. These properties are distinguished from public ownership, moral desirability or government provision; the classification concerns how access and use interact." }, { "id": 3876, "name": "commons overuse dilemma", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "A pattern in which individual benefits from using a shared, subtractable resource can encourage cumulative extraction that damages the common resource. Such a dilemma depends on incentives and governance; it is not an unavoidable fate of every collectively managed commons." }, { "id": 4163, "name": "possible world", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A complete consistent way things could be, used as a semantic alternative in modal reasoning. Formal use can evaluate what is possible, necessary or compatible with information without committing to the physical or metaphysical existence of other worlds." }, { "id": 4318, "name": "public reason", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The idea that shared political rules should be justifiable to those over whom they claim authority through reasons they can reasonably assess. Versions differ about acceptable reasons and scope; it is not merely any statement made in public or a demand for unanimous belief." }, { "id": 5099, "name": "economic property rights", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 3 ], "definition": "Institutionally recognized claims concerning use, benefits and transfer of a resource. These claims may be divided among parties and depend on enforcement; ownership form is distinct from a good's physical rivalry or ease of exclusion." }, { "id": 5100, "name": "common ownership", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "Holding assets for an organization or community without dividing ownership into individually held shares. Rules of membership, use and decision-making may still govern access; common ownership does not automatically mean unrestricted open access." }, { "id": 5101, "name": "public ownership", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "Ownership of assets or enterprises by a government or public body at a specified level. It concerns who owns or controls the assets, distinct from whether their outputs are public goods or whether management is democratic, commercial or effective." }, { "id": 5264, "name": "co-production of public services", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Development or delivery of public services through contributions from both service organizations and people using or affected by them. It extends beyond consultation to participation in activities such as design, delivery or management." }, { "id": 5975, "name": "evaporative cooling", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Cooling produced when liquid evaporation draws energy from its surroundings. For direct air–water systems the available wet-bulb depression limits performance, and moisture is added to the air; indirect arrangements separate the treated air from evaporation." } ] }, { "task": "B12", "split": "final", "origin": "public-human-question", "domain": "economics", "question": "Under perfect information and negligible administrative burdens, when are a carbon tax and an auctioned cap-and-trade system equivalent across time? Compare permits that expire each year with permits that can be banked or borrowed. Explain how timing, borrowing constraints, and uncertainty affect efficiency and whether total permits alone determine the resulting allocation.", "requirements": [ "State the conditions supporting a tax-permit equivalence.", "Compare annual expiration with banking and borrowing.", "Explain the roles of intertemporal choices and borrowing constraints.", "Assess the claim that only total lifetime permits matter." ], "poolSize": 13, "poolAttainableObligations": [], "poolMissingObligations": [ 0, 1, 2, 3 ], "metrics": { "dense": { "top5": [ 3222, 4868, 3876, 3079, 2885 ], "ndcg5": 0.70807361818795, "agreedObligationCoverage5": 0.0, "coveredObligations": [], "agreedAssignments": 0, "labelRedundancy": 0.0, "embeddingRedundancy": 0.3458053779936369, "documentsWithoutAgreedObligations": 5, "increments": [ { "id": 3222, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4868, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3876, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3079, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2885, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3222, 4868, 3876, 3079, 2885 ], "ndcg5": 0.70807361818795, "agreedObligationCoverage5": 0.0, "coveredObligations": [], "agreedAssignments": 0, "labelRedundancy": 0.0, "embeddingRedundancy": 0.3458053779936369, "documentsWithoutAgreedObligations": 5, "increments": [ { "id": 3222, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4868, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3876, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3079, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2885, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3222, 5094, 3876, 3079, 814 ], "ndcg5": 0.7726974724935582, "agreedObligationCoverage5": 0.0, "coveredObligations": [], "agreedAssignments": 0, "labelRedundancy": 0.0, "embeddingRedundancy": 0.20019290315558944, "documentsWithoutAgreedObligations": 5, "increments": [ { "id": 3222, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5094, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3876, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3079, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 814, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3079, 2885, 3222, 3876, 5093 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.0, "coveredObligations": [], "agreedAssignments": 0, "labelRedundancy": 0.0, "embeddingRedundancy": 0.29420849015723605, "documentsWithoutAgreedObligations": 5, "increments": [ { "id": 3079, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2885, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3222, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3876, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5093, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 1287, "coverageMaximizingSubsets": 1287, "coveredObligations": 0 }, "poolItems": [ { "id": 814, "name": "forgotten Zion answered through maternal and engraved images", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Zion’s claim of abandonment receives an answer comparing divine remembrance with a mother’s attachment and then with an inscription upon palms. The imagery addresses threatened memory and attachment while refusing to let even the possible failure of maternal remembrance define the promise." }, { "id": 1794, "name": "penal operation behind a protective administrative label", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Examining whether a system imposes accusation, compulsory control and penalties even when presented as protection or regulation. Its operative powers, rather than its reassuring name alone, determine the procedural questions to ask." }, { "id": 2545, "name": "System Year as a Galactic model convention", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A proposed label for solar-system Galactic phase or duration under a named motion model and epoch. Its parameters and uncertainty remain attached; an approximate Galactic circuit does not become an exact metrological year or a universal calendar merely by receiving a name." }, { "id": 2752, "name": "partition-time consistency trade-off", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "In a distributed data system subject to a network partition, the CAP result rules out simultaneously guaranteeing its specified strong consistency and availability for all requests. The conflict is conditional on partition; it is not a universal instruction to choose only two desirable properties at all times." }, { "id": 2885, "name": "carbon footprint", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Greenhouse-gas emissions attributed to an activity, product or organization over a declared boundary and period, commonly expressed in carbon-dioxide-equivalent units. A comparison also requires a functional unit and stated accounting choices, rather than an unexplained total." }, { "id": 3079, "name": "trade-off under constraints", "grade": 1.5, "judgeA": 2, "judgeB": 1, "agreedObligations": [], "definition": "A situation in which improving one relevant property, outcome or allocation entails reducing another within stated constraints. The competing dimensions and feasible alternatives must be specified; a trade-off observed under one design or environment need not be permanent." }, { "id": 3080, "name": "life-history allocation", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An evolutionary framework relating an organism's schedule of growth, maturation, reproduction and survival to ecological conditions and constraints on resource allocation. It investigates patterns and trade-offs rather than attributing deliberate strategic calculation to every organism." }, { "id": 3222, "name": "capital substitution in sustainability", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The distinction between approaches allowing losses of natural assets to be offset by other forms of capital and approaches requiring certain critical natural assets or functions to be preserved separately. The dispute concerns substitutability and measurement, not simply caring more or less." }, { "id": 3876, "name": "commons overuse dilemma", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A pattern in which individual benefits from using a shared, subtractable resource can encourage cumulative extraction that damages the common resource. Such a dilemma depends on incentives and governance; it is not an unavoidable fate of every collectively managed commons." }, { "id": 4868, "name": "linear resource type", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A type discipline requiring a tracked resource to be used exactly once in the relevant ownership interpretation, controlling duplication and disposal." }, { "id": 5093, "name": "economic tax incidence", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The distribution of a tax's economic burden after changes in prices, wages or other behavior are considered. It differs from statutory responsibility for remitting the tax, and depends on market responses and the time horizon." }, { "id": 5094, "name": "statutory tax incidence", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The legal assignment of responsibility for paying or remitting a tax to the collecting authority. It specifies an administrative obligation rather than determining who ultimately bears the economic cost." }, { "id": 5098, "name": "Coase theorem", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A bargaining result under idealized conditions, including defined rights and negligible transaction costs, in which parties can reach an efficient allocation despite an externality. Initial rights still affect distribution; the result does not imply that real bargaining is costless or that fairness is settled." } ] }, { "task": "B13", "split": "final", "origin": "prospective-ai-task", "domain": "philosophy", "question": "I coordinate a small disaster-relief charity. We promised recurring deliveries to one village, but a newly isolated village now has greater unmet need. Our funds cannot fully serve both, and neither group caused the shortage. Recommend a defensible allocation procedure that weighs promise keeping, equal treatment, and preventing harm while making uncertainty and the reasons owed to both communities explicit.", "requirements": [ "Distinguish the force of a prior promise from impartial need.", "Compare at least two defensible allocation principles.", "Address uncertain information and the possibility of revising the allocation.", "Specify how to justify the decision to both affected communities." ], "poolSize": 14, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 1836, 2247, 1675, 595, 874 ], "ndcg5": 0.6940209635364101, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.43479864695089915, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 1836, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 2247, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 1675, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 595, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 874, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 1836, 2247, 1675, 595, 874 ], "ndcg5": 0.6940209635364101, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.43479864695089915, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 1836, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 2247, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 1675, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 595, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 874, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 1675, 4191, 1647, 979, 385 ], "ndcg5": 0.5765835880373428, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 2 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.23625784920334478, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 1675, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 4191, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1647, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 979, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 385, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [ 3 ], "uniqueObligationsGainedVsDense": [ 0 ], "lostCountVsDense": 1, "gainedCountVsDense": 1 }, "oracle": { "top5": [ 979, 1675, 2247, 1836, 2167 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.5, "embeddingRedundancy": 0.432510974933385, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 979, "agreedObligations": [ 0, 1, 2 ], "newObligationsAtRank": [ 0, 1, 2 ] }, { "id": 1675, "agreedObligations": [ 1 ], "newObligationsAtRank": [] }, { "id": 2247, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 1836, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 2167, "agreedObligations": [ 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 0 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 2002, "coverageMaximizingSubsets": 385, "coveredObligations": 4 }, "poolItems": [ { "id": 385, "name": "Joshua: promised land and the cost of possession", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A reading frame that connects conquest, allotment and covenant choice with the narrative's claim of fulfilled promise, retaining both reported mass killing and the continuing presence of other peoples within the land." }, { "id": 511, "name": "the promised garden's unspoiled provision", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The Qur'anic promise of gardens described through incorruptible water, unchanging milk, pleasing wine, clarified honey and fruit. Concrete abundance makes the hoped-for future perceptible while keeping it distinct from the decay and loss of present provision." }, { "id": 595, "name": "designated recipients of obligatory alms", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An allocation of alms names the poor, needy, collectors, people whose hearts are won, ransoming, debtors, God’s cause and the wayfarer. The distribution includes material relief and communal purposes rather than one undifferentiated act of generosity." }, { "id": 760, "name": "promised possession deferred beyond one lifetime", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The covenant scene in Genesis places promised land beyond Abram’s own peaceful death and an intervening period of affliction. The delay is also connected with a stated limit not yet reached in the Amorite’s iniquity, making promise distinct from immediate authorization to possess." }, { "id": 874, "name": "Unified Escrow for verified non-humanitarian tranches", "grade": 1.5, "judgeA": 2, "judgeB": 1, "agreedObligations": [], "definition": "A proposed multi-donor fund holds conditional reconstruction or sector-specific benefits until agreed milestones are certified. Public rules, donor oversight and audit make release predictable, while the draft intends lifesaving humanitarian flows to remain outside the withholding mechanism." }, { "id": 979, "name": "rerouting assistance toward unmet essential-service deficits", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 2 ], "definition": "A draft corrective response that sends fuel or food toward the services and districts missing essential floors instead of making deprivation the penalty for a missed target. Outcome monitoring guides rerouting; proposed measures against a responsible node remain separate from care owed to the population." }, { "id": 1647, "name": "relief through payment without acceptance of a rightful claim", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A person’s gratitude for release obtained by payment coexisting with rejection of the alleged entitlement for which money was demanded." }, { "id": 1675, "name": "equal per-person funding below a viable service threshold", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "A formally proportional allocation that leaves a small or dispersed group unable to obtain a functioning service because essential fixed costs are not covered." }, { "id": 1836, "name": "relief expenditure separated from relief reaching recipients", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "Public money nominally assigned to relieving poverty can be consumed by litigation or administration that burdens the intended recipients. Total spending and actual alleviation require separate accounting." }, { "id": 2167, "name": "material charity without recognition as a fellow participant", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "A setting permits assistance to people in need while refusing equal social encounter or cooperation with those same people as peers. Receiving a benefit and being recognized as a participant remain different relations." }, { "id": 2247, "name": "relief for one excluded person charged to other excluded people", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "An authority remedies one person’s exclusion by transferring costs or resources from another disadvantaged group without that group’s effective participation. The claimant’s possible entitlement and the justice of this particular allocation remain separate questions." }, { "id": 3891, "name": "principal–agent problem", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A difficulty arising when an agent acts on a principal's behalf while their interests or information differ and the principal cannot fully observe or control performance. Delegation creates a relationship in which apparently compliant action may diverge from the principal's purpose." }, { "id": 3948, "name": "financial discounting", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Converting cash flows at different dates to a common valuation date through an explicit rate or discount-factor convention. The procedure makes timing comparable within a model; its result depends on the selected rates, horizon and treatment of uncertainty." }, { "id": 4191, "name": "allocation concealment", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Keeping upcoming trial assignments unavailable to those who could influence enrollment or allocation, so anticipation of a group does not shape who receives it. This protects the allocation process and is distinct from blinding participants or assessors after assignment." } ] }, { "task": "B14", "split": "final", "origin": "prospective-ai-task", "domain": "philosophy", "question": "I am an oral-history archivist deciding whether to label a disputed local event as established, probable, or unresolved. Several witnesses repeat one account, an older written record contradicts it, and some witnesses may have heard the story from each other. Propose a method for weighing testimony, source independence, and documentary evidence without equating either age or majority agreement with truth.", "requirements": [ "Separate number of witnesses from independence of their evidence.", "Compare possible strengths and weaknesses of testimony and records.", "Define evidence-sensitive confidence labels.", "State what new evidence would justify changing the archive's label." ], "poolSize": 12, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 125, 2128, 1080, 1715, 5028 ], "ndcg5": 0.7014941890986883, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.42857142857142855, "embeddingRedundancy": 0.4174922296135282, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 125, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 1, 2 ] }, { "id": 2128, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0 ] }, { "id": 1080, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1715, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 5028, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 3 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 125, 2128, 1080, 1715, 5028 ], "ndcg5": 0.7014941890986883, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.42857142857142855, "embeddingRedundancy": 0.4174922296135282, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 125, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 1, 2 ] }, { "id": 2128, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0 ] }, { "id": 1080, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1715, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 5028, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 3 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 125, 1080, 1757, 3994, 70 ], "ndcg5": 0.4189377179506712, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 1, 2 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.21204750345772144, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 125, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 1, 2 ] }, { "id": 1080, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1757, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3994, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 70, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [ 0, 3 ], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 2, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 125, 3845, 5026, 5028, 2128 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 10, "labelRedundancy": 0.6, "embeddingRedundancy": 0.40938067877089396, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 125, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 1, 2 ] }, { "id": 3845, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [] }, { "id": 5026, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 5028, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [] }, { "id": 2128, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 792, "coverageMaximizingSubsets": 204, "coveredObligations": 4 }, "poolItems": [ { "id": 36, "name": "the reproducible handoff", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "a portable record of inputs, method and version, expected output, and limits, sufficient for another reader to repeat a calculation or check a passage through the carrier. A mismatch remains visible. Agreement records a successful repeat under stated conditions; it does not certify every assumption." }, { "id": 70, "name": "history-of-the-universe learning", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "Teaching science as a connected evidence-based story from cosmic origins through matter, life, intelligence, civilization, and possible futures, while distinguishing established knowledge from speculation." }, { "id": 125, "name": "textual attestation distinguished from historical proof", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2 ], "definition": "A reading distinction between establishing that a source makes a claim and establishing that the claimed event occurred as described. Verified wording, narrative confidence and a work's stated purpose remain evidence about the text before further historical inferences are made." }, { "id": 1080, "name": "international-bench truth record with victim testimony", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The draft proposes a Truth and Reconciliation Commission to document violations, hear victims, obtain records and testimony, and produce a public account. It places this immediate truth function alongside criminal proceedings rather than treating the commission’s record as the only form of accountability." }, { "id": 1198, "name": "testimonial exclusion by status", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "A rule or practice that denies a person’s account evidentiary standing because of an assigned social status, even where that person could directly know the event. Lack of admitted testimony must then be distinguished from lack of witnesses or knowledge." }, { "id": 1715, "name": "disagreement revealing the structure of a public question", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "Contrary accounts can expose the parties, stakes and assumptions of a controversy even when the observer cannot yet decide between them. Learning what is disputed is distinct from resolving the dispute." }, { "id": 1757, "name": "confidence as a condition of received correction", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "An account in which prior confidence affects whether a challenge is heard as helpful remonstrance or hostile attack. Confidence can enable correction without by itself deciding that the challenge is true." }, { "id": 2128, "name": "a meeting deciding not to hear the disputed evidence", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 2 ], "definition": "Use of an assembly’s procedures to prevent presentation of the subject supposedly under discussion. A vote can settle who speaks without settling the truth of the excluded claim." }, { "id": 3845, "name": "testimony as an attestation", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2 ], "definition": "A communicated attestation offered as evidence about a matter for others to consider. The existence and wording of a testimony, the witness's credibility and the truth of the attested matter are distinguishable questions." }, { "id": 3994, "name": "freedom of religion or belief", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Freedom of persons and communities to hold and manifest religion or belief, together with freedom not to profess or practice a religion. The protected choice is distinguished from a finding that any particular belief is true or institution authoritative." }, { "id": 5026, "name": "oral history", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "Collection and interpretation of accounts of the past through planned interviews with people recalling experiences or observations. It preserves perspectives and memories as sources while requiring attention to interview context, later recollection and corroboration." }, { "id": 5028, "name": "historical method", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "Practices for identifying, evaluating and relating sources to construct accounts of past events and settings. Written, oral and material evidence can contribute, with methods adapted to the question and surviving record rather than one universally sufficient checklist." } ] }, { "task": "B15", "split": "final", "origin": "prospective-ai-task", "domain": "philosophy", "question": "I curate a museum boat whose wooden parts were gradually replaced during restoration. Most visitors value continuity with its working history, while a donor insists that authenticity requires original material. We must write an honest exhibit label and decide which future repairs to permit. Compare material, historical, and functional accounts of identity, and explain how a practical preservation policy can follow without pretending the philosophical dispute is settled.", "requirements": [ "Distinguish numerical identity, historical continuity, and authenticity.", "Compare at least two accounts using the repaired boat.", "Propose wording principles that represent the uncertainty honestly.", "Derive a practical repair policy and identify its value commitments." ], "poolSize": 16, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 4078, 983, 5042, 4079, 4081 ], "ndcg5": 0.8066031581233893, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.5, "embeddingRedundancy": 0.5057975898092599, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 4078, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 983, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 5042, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4079, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2 ] }, { "id": 4081, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 4078, 983, 5042, 4079, 4081 ], "ndcg5": 0.8066031581233893, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.5, "embeddingRedundancy": 0.5057975898092599, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 4078, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 983, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 5042, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4079, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2 ] }, { "id": 4081, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 2151, 2454, 4078, 4851, 2062 ], "ndcg5": 0.5505982524946987, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.19338940039014588, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 2151, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 2454, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4078, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 4851, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2062, "agreedObligations": [ 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 983, 2062, 4078, 4081, 2151 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 9, "labelRedundancy": 0.5555555555555556, "embeddingRedundancy": 0.3613438100535021, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 983, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 2062, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 4078, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 4081, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2 ] }, { "id": 2151, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 4368, "coverageMaximizingSubsets": 1155, "coveredObligations": 4 }, "poolItems": [ { "id": 961, "name": "case continuity for missing persons and remains", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The draft gives each missing-person case a continuing pathway from source records and family information to identification, notification and dignified return. Uncertainty remains visible while work proceeds; confirmed identity, family communication and release of remains are not replaced by a bargaining ratio." }, { "id": 983, "name": "documented and reversible conservation work", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "A draft heritage practice requiring justified interventions, public works records and reversibility where feasible, with emergency readiness and later damage assessment. Conserving a site is distinguished from using works to establish unilateral territorial or custodial changes." }, { "id": 1291, "name": "selective preservation in a historical material record", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Unequal chances that past organisms or events leave recoverable traces, depending on material properties, place, burial, later destruction and collection, so the recovered sample is not an undistorted census of the past." }, { "id": 2062, "name": "policy choice presented as deductive continuity", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "A decision that appears to follow formally from inherited rules while also selecting among views of what would serve the community. Making that choice visible permits its grounds to be examined rather than treating precedent as the whole explanation." }, { "id": 2151, "name": "practical valuation distinguished from exact historical separation", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "A valuation method can seek a useful separation of presently identifiable components without claiming to reconstruct every historical contribution exactly. The operational boundary and tolerated imprecision should be stated rather than hidden behind a theoretically pure category." }, { "id": 2208, "name": "measurement records accompanied by preserved actual units", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Passing measurements to future readers requires preserving or recoverably specifying the units actually used. Numbers without continuity of their reference standard transmit only part of the observation, even if the original measurement was careful." }, { "id": 2442, "name": "historical description, current allocation", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "The distinction between an earlier description retained as history and the validated registry governing present use. A linked amendment identifies the changed assignment and its reason without silently rewriting the earlier wording. Historical references remain tied to their original snapshots; preserving a statement does not renew an obsolete allocation." }, { "id": 2454, "name": "a missing distinction with preserved expression witnesses", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A proposed vocabulary gap is supported by retained attempts to express a particular distinction, including existing concepts tried, their failed or misleading readings, and the new boundary required. Low connectivity alone is not evidence that a concept is missing." }, { "id": 3810, "name": "epistemology", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Philosophical inquiry into knowledge, justification, their sources and their limits. It examines relations among belief, truth, evidence, experience and practical understanding without treating one theory of their connection as already settled." }, { "id": 4078, "name": "cultural heritage", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 3 ], "definition": "Material and living legacies selected and recognized as significant for a community's continuity and identity. Heritage is distinguished from everything that happens to survive from the past, because selection and valuation shape what is preserved." }, { "id": 4079, "name": "archive", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "A preserved body of records retained for enduring evidentiary, historical or cultural value, often connected through the activities that produced them. An archive's selection, arrangement and provenance shape what later readers can inspect." }, { "id": 4080, "name": "museum", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "An institution that collects, preserves, studies and interprets objects or other material of cultural or scientific significance. Public display is one function among others, and an exhibition represents a selection from the institution's holdings and interpretive choices." }, { "id": 4081, "name": "curatorship", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "Stewardship that selects, documents, interprets and cares for a collection or presentation. Curatorial choices mediate what others encounter, so preservation, explanation and selection are distinguishable responsibilities rather than neutral storage alone." }, { "id": 4082, "name": "public history", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "Practice that uses historical methods to engage publics beyond specialized academic settings, through places, collections, media and shared interpretation. It connects evidentiary inquiry with how communities encounter and use accounts of the past." }, { "id": 4851, "name": "compare-and-swap", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An atomic read–modify–write operation replacing a memory value only if it equals an expected value, and reporting whether the replacement occurred." }, { "id": 5042, "name": "cultural-property repatriation", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Return of cultural objects or remains to a place, community, owner or successor with a recognized or asserted connection. Claims can involve acquisition history, law and ethics, and require identifying the object, claimant and grounds rather than presuming every case identical." } ] }, { "task": "B16", "split": "development", "origin": "prospective-ai-task", "domain": "philosophy", "question": "I edit a science exhibition comparing two models of animal group behavior. One predicts movement accurately but offers little interpretable mechanism; the other describes individual interactions clearly but predicts less well outside familiar conditions. We can feature only one model prominently. Recommend how to assess explanatory value, predictive success, and scope, and what caveats the exhibit should communicate to a general audience.", "requirements": [ "Distinguish successful prediction from a causal or mechanistic explanation.", "Compare the models using more than one scientific aim.", "Account for their different ranges of applicability.", "Recommend a presentation choice and appropriate caveats." ], "poolSize": 18, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 2371, 5854, 5307, 4980, 2226 ], "ndcg5": 0.21027316506406984, "agreedObligationCoverage5": 0.0, "coveredObligations": [], "agreedAssignments": 0, "labelRedundancy": 0.0, "embeddingRedundancy": 0.3854511382094291, "documentsWithoutAgreedObligations": 5, "increments": [ { "id": 2371, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5854, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5307, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4980, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2226, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 2371, 5854, 5307, 4980, 2226 ], "ndcg5": 0.21027316506406984, "agreedObligationCoverage5": 0.0, "coveredObligations": [], "agreedAssignments": 0, "labelRedundancy": 0.0, "embeddingRedundancy": 0.3854511382094291, "documentsWithoutAgreedObligations": 5, "increments": [ { "id": 2371, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5854, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5307, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4980, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2226, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 2226, 3497, 5310, 2371, 3249 ], "ndcg5": 0.5798609372556051, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.221660232804054, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 2226, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3497, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 1, 2, 3 ] }, { "id": 5310, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2371, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3249, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 0, 1, 2, 3 ], "lostCountVsDense": 0, "gainedCountVsDense": 4 }, "oracle": { "top5": [ 3249, 3497, 2242, 3100, 3475 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 11, "labelRedundancy": 0.6363636363636364, "embeddingRedundancy": 0.15963469788049062, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3249, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 3497, "agreedObligations": [ 1, 2, 3 ], "newObligationsAtRank": [ 2 ] }, { "id": 2242, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] }, { "id": 3100, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 3475, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 0, 1, 2, 3 ], "lostCountVsDense": 0, "gainedCountVsDense": 4 } }, "oracleEnumeration": { "subsetsEnumerated": 8568, "coverageMaximizingSubsets": 3354, "coveredObligations": 4 }, "poolItems": [ { "id": 2226, "name": "spreading a brief action over time to make it inspectable", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A technique redistributes a rapid intense action into a sustained controllable process, making effects available for sequential observation and intervention. Extending access can reveal relations hidden when the effects were crowded into an instant." }, { "id": 2242, "name": "exhibition work substituted for ordinary institutional performance", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "An institution displays work specially prepared for visitors as if it represented ordinary practice or lasting outcomes. Inspection can then certify the staged sample while leaving typical conditions and the recipients’ later experience unexamined." }, { "id": 2371, "name": "association judged by interaction rather than group size", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "A gathering’s collective character depends on how participants interact, not its numbers or formal organization. Shared suggestion can govern a small meeting or organized union, while reciprocal development of differences can occur in larger institutions; actual gatherings can mix these modes." }, { "id": 2830, "name": "model predictive control", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Control that repeatedly predicts behaviour over a finite horizon, optimizes an input sequence under constraints, applies its first part and then recomputes from updated information. Its predictions and feasible actions depend on the model and available measurements." }, { "id": 2877, "name": "ergonomics", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Study and design of interactions between people and other system elements to support human well-being and effective performance. It considers physical, cognitive and organizational conditions rather than treating every error as an individual person's deficiency." }, { "id": 3100, "name": "population dynamics", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "Changes in the size and composition of a population through time, studied through processes and models of birth, death, movement and interaction. A model specifies which processes and environmental conditions it includes." }, { "id": 3249, "name": "abductive reasoning", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 3 ], "definition": "Reasoning that proposes an explanation for an observation, often by comparing candidates for how well they account for it. The selected explanation remains defeasible; explanatory fit alone does not make it the only possible cause." }, { "id": 3475, "name": "randomized controlled trial", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 2 ], "definition": "An experiment assigning eligible units to interventions by a random mechanism and comparing outcomes. Randomization limits certain confounding in expectation, but attrition, nonadherence, measurement and limited scope can still affect interpretation." }, { "id": 3497, "name": "model selection", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2, 3 ], "definition": "Choosing among candidate models using criteria such as fit, complexity, predictive performance or prior support. The selected model is best only relative to the considered set, criterion and evidence." }, { "id": 3717, "name": "observational learning", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "Learning that occurs through observing other agents and their behavior. Acquisition through a model can be distinguished from direct reinforcement of the learner, and observing a behavior does not require that the model intentionally teaches it." }, { "id": 3855, "name": "scientific realism", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 1, 3 ], "definition": "The view that successful science can give approximately true knowledge of a world independent of our perceptions, including some unobservable entities or processes. It treats explanatory and predictive success as relevant to that commitment, not as infallibility of every current theory." }, { "id": 4080, "name": "museum", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "An institution that collects, preserves, studies and interprets objects or other material of cultural or scientific significance. Public display is one function among others, and an exhibition represents a selection from the institution's holdings and interpretive choices." }, { "id": 4276, "name": "wahdat al-wujud", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A doctrine in Islamic philosophy and Sufi interpretation identifying one divine existence through which particular beings exist. Its accounts and legitimacy are disputed; unity of existence is not safely paraphrased as every finite thing simply being God or as all Muslims sharing the doctrine." }, { "id": 4979, "name": "bystander effect", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A context-dependent tendency for an individual's probability of helping to decrease when other potential helpers are present. It concerns individual response under specified conditions and need not mean that a larger group is less likely to provide any help at all." }, { "id": 4980, "name": "social facilitation", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "A change in task performance associated with the presence of others as an audience or fellow performers. Familiar or dominant responses may be strengthened while difficult or unfamiliar performance may worsen; presence does not uniformly improve ability." }, { "id": 5307, "name": "ethnomethodology", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "Study of the practical methods people use to make everyday activities intelligible and orderly to one another. It examines how recognizable social situations are accomplished in interaction rather than assuming their order is simply given in advance." }, { "id": 5310, "name": "back-stage interaction", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "In a dramaturgical account, activity outside the relevant audience's view where a public performance may be prepared, adjusted or relaxed. The boundary depends on the audience and situation rather than one permanently private physical place." }, { "id": 5854, "name": "ecological facilitation", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An interaction in which one organism improves conditions or performance for another without a demonstrated negative effect on the facilitator under the stated conditions." } ] }, { "task": "B17", "split": "final", "origin": "prospective-ai-task", "domain": "biology", "question": "I manage two isolated reserves containing the same threatened animal. One population is small and closely related; the other occupies a different climate and appears locally adapted. A corridor could allow breeding and seasonal movement but also spread disease. Help decide whether to build it now, stage its opening, or gather more evidence, distinguishing demographic rescue, genetic effects, and ecological risks.", "requirements": [ "Compare demographic and genetic benefits of connectivity.", "Distinguish inbreeding concerns from risks to local adaptation.", "Consider disease movement and habitat use.", "Give a conditional decision with monitoring and revision criteria." ], "poolSize": 14, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3108, 3109, 5779, 3103, 1824 ], "ndcg5": 0.6072827569747501, "agreedObligationCoverage5": 0.25, "coveredObligations": [ 2 ], "agreedAssignments": 2, "labelRedundancy": 0.5, "embeddingRedundancy": 0.46870010399387996, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3108, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 3109, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5779, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3103, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1824, "agreedObligations": [ 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3108, 3109, 5779, 3103, 1824 ], "ndcg5": 0.6072827569747501, "agreedObligationCoverage5": 0.25, "coveredObligations": [ 2 ], "agreedAssignments": 2, "labelRedundancy": 0.5, "embeddingRedundancy": 0.46870010399387996, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3108, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 3109, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5779, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3103, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1824, "agreedObligations": [ 2 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 1824, 4201, 5507, 3108, 36 ], "ndcg5": 0.4993571760837298, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 2 ], "agreedAssignments": 3, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.13828950135614812, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 1824, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 4201, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5507, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] }, { "id": 3108, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 36, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 0 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 }, "oracle": { "top5": [ 3108, 5773, 3082, 5507, 5768 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.3436105460283355, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3108, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 5773, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] }, { "id": 3082, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 1 ] }, { "id": 5507, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 5768, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 0, 1, 3 ], "lostCountVsDense": 0, "gainedCountVsDense": 3 } }, "oracleEnumeration": { "subsetsEnumerated": 2002, "coverageMaximizingSubsets": 100, "coveredObligations": 4 }, "poolItems": [ { "id": 36, "name": "the reproducible handoff", "grade": 0.5, "judgeA": 0, "judgeB": 1, "agreedObligations": [], "definition": "a portable record of inputs, method and version, expected output, and limits, sufficient for another reader to repeat a calculation or check a passage through the carrier. A mismatch remains visible. Agreement records a successful repeat under stated conditions; it does not certify every assumption." }, { "id": 1824, "name": "habitat alteration changing relations among survivors", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "Changing a habitat can remove some occupants while altering the relationships among those that remain. Counting losses alone does not capture the reorganization of the continuing community." }, { "id": 2913, "name": "nature-based solutions", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "Actions that protect, restore or manage ecosystems to address societal challenges while seeking benefits for biodiversity and people. Their effectiveness depends on ecological processes and local conditions, and should be evaluated alongside social effects and technical alternatives." }, { "id": 3082, "name": "ecological niche", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 1, 2 ], "definition": "The conditions, resources and interactions under which a population persists and the role it plays within its environment. Different ecological theories emphasize different aspects of this concept, so an account should state whether it concerns requirements, realized use or effects on others." }, { "id": 3103, "name": "ecological succession", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Change in the composition and structure of an ecological community through time, including colonization of new habitat and change following disturbance. Patterns can recur without requiring every site to follow one inevitable sequence toward a fixed final community." }, { "id": 3108, "name": "ecological corridor", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2 ], "definition": "A habitat connection that enables organisms, or biological materials carried by them, to move between otherwise separated areas. Its usefulness depends on the species, scale and conditions of movement rather than on visual continuity alone." }, { "id": 3109, "name": "conservation biology", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "An interdisciplinary field concerned with understanding and limiting biodiversity loss, combining ecological and evolutionary evidence with the social conditions of protection and resource use. Its goals involve values as well as empirical claims." }, { "id": 4201, "name": "criterion-related validity", "grade": 0.5, "judgeA": 0, "judgeB": 1, "agreedObligations": [], "definition": "Evidence about how a measure relates to a relevant external outcome or criterion. Concurrent assessment compares at the same time; predictive assessment concerns later outcomes, and evidence for one cannot automatically substitute for the other." }, { "id": 5493, "name": "genetic chimerism", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An organism containing cells of different genetic origins, including animal cases formed by embryo aggregation or acquisition of another individual’s cells through transplantation." }, { "id": 5504, "name": "genetic counseling", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A process combining interpretation of family and medical information, education, and support to help people understand genetic contributions to disease and make informed choices about their implications." }, { "id": 5507, "name": "genetic population structure", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 0 ], "definition": "Systematic differences in allele frequencies among subpopulations, arising through processes such as non-random mating, limited movement, drift, or demographic history." }, { "id": 5768, "name": "population viability analysis", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "A species- or population-specific assessment estimating persistence or extinction risk over a stated period under demographic, environmental and management assumptions." }, { "id": 5773, "name": "ecological rescue effect", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0 ], "definition": "Reduction of local population extinction risk through immigration from other populations, linking persistence to movement and the condition of surrounding sources." }, { "id": 5779, "name": "ecological resource partitioning", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "Differentiation in resource use among coexisting organisms, across resource type, space or time, that can reduce overlap in limiting-resource demands." } ] }, { "task": "B18", "split": "final", "origin": "prospective-ai-task", "domain": "biology", "question": "I am a lake-restoration officer choosing between reducing nutrient inflow, removing a common fish, or combining the two. The lake is persistently cloudy, but fish surveys and nutrient measurements vary strongly by season. Explain how food-web interactions, external inputs, and delayed ecosystem responses could support different choices, and propose monitoring that can separate treatment effects from ordinary seasonal variation.", "requirements": [ "Compare the candidate interventions through plausible ecological pathways.", "Separate external nutrient inputs from effects within the food web.", "Address delayed responses or feedbacks.", "Design a comparison and monitoring schedule that handles seasonality." ], "poolSize": 16, "poolAttainableObligations": [ 0, 1, 3 ], "poolMissingObligations": [ 2 ], "metrics": { "dense": { "top5": [ 3111, 3090, 3120, 75, 3077 ], "ndcg5": 0.47697017530379393, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 3, "labelRedundancy": 0.0, "embeddingRedundancy": 0.36304879140716934, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3111, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3090, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 3120, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 75, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3077, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3111, 3090, 3120, 75, 3121 ], "ndcg5": 0.47697017530379393, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 1 ], "agreedAssignments": 4, "labelRedundancy": 0.5, "embeddingRedundancy": 0.4122163174148422, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3111, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3090, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 3120, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 75, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3121, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [ 3 ], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 1, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3077, 3121, 3111, 2740, 1970 ], "ndcg5": 0.6038946136966565, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 3, "labelRedundancy": 0.0, "embeddingRedundancy": 0.12691770248564566, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3077, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3121, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 3111, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2740, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1970, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3077, 3090, 3121, 4387, 3117 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.625, "embeddingRedundancy": 0.3790019126314489, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3077, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3090, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 3121, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 4387, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 3117, "agreedObligations": [ 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 4368, "coverageMaximizingSubsets": 2155, "coveredObligations": 3 }, "poolItems": [ { "id": 75, "name": "closed-loop habitat", "grade": 1.5, "judgeA": 2, "judgeB": 1, "agreedObligations": [], "definition": "A habitat designed by treating air, water, food, waste, energy, organisms, and maintenance as coupled cycles whose inputs, outputs, losses, and dependencies are explicit." }, { "id": 1124, "name": "separate measures for infrastructure ecology and heritage harm", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "A draft monitoring agenda keeps power capacity loss, pollution, ecological damage and cultural-site damage as different kinds of impact. Each needs appropriate units, baselines and evidence; a shared dashboard must not make unlike measurements interchangeable or infer a common attribution from proximity." }, { "id": 1558, "name": "seasonal variation recorded in sediment layers", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "A proposed relation between recurring changes in water volume and transport and alternating layers of coarser and finer deposited material." }, { "id": 1970, "name": "internal mutual support alongside external subjection", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An institution may support its own members while collectively dominating outsiders. Internal reciprocity and the treatment of nonmembers therefore require separate evaluation." }, { "id": 2442, "name": "historical description, current allocation", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The distinction between an earlier description retained as history and the validated registry governing present use. A linked amendment identifies the changed assignment and its reason without silently rewriting the earlier wording. Historical references remain tied to their original snapshots; preserving a statement does not renew an obsolete allocation." }, { "id": 2740, "name": "memoization", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Reusing stored results when the same function inputs recur, avoiding repeated computation at the cost of storage. The stored result must remain valid for all relevant dependencies; hidden state or side effects can make apparently identical calls require different treatment." }, { "id": 3077, "name": "environmental monitoring", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "Repeated or organized observation of environmental conditions using a design suited to a stated question. It can establish baselines and trends, while requiring attention to sampling, measurement and interpretation before change or its cause is inferred." }, { "id": 3090, "name": "nutrient cycle", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "Movement and transformation of biologically relevant matter among organisms and environmental reservoirs. Materials can be reused through linked processes; this cycling differs from energy flowing through a system and becoming less available for work." }, { "id": 3110, "name": "ecological restoration", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "Assisting the recovery of an ecosystem that has been damaged, degraded or destroyed through interventions guided by ecological understanding and explicit goals. Recovery need not mean exact reproduction of one historical condition." }, { "id": 3111, "name": "ecosystem service", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A benefit people obtain from ecosystems, such as food, water regulation, pollination or cultural experience. This human-benefit lens helps describe dependencies while leaving room for ecological and ethical values not captured by service accounting." }, { "id": 3117, "name": "ecosystem", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "Organisms and their physical environment considered as an interacting system, connected by transfers of energy and cycling of materials. Its boundary and scale depend on the question being studied; it is more than a list of species sharing a place." }, { "id": 3120, "name": "net ecosystem production", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Gross primary production minus respiration by all organisms within the selected ecosystem and time interval. Other transfers and disturbances must be accounted for before interpreting this difference as a complete carbon-stock change." }, { "id": 3121, "name": "food web", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "A network of feeding relationships that connects organisms through transfers of matter and energy. A participant may draw on several food sources and occupy more than one trophic level, so a single food chain captures only selected paths through the network." }, { "id": 4387, "name": "trophic cascade", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "An indirect ecological effect transmitted across trophic levels, for example when changing predators changes herbivore pressure on producers. Direction and strength depend on the food web and conditions; restoring one species need not guarantee a desired ecosystem outcome." }, { "id": 5801, "name": "wetland", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "An ecosystem whose sustained or recurrent inundation or saturation shapes soil processes and biota, often producing oxygen-limited soils and water-adapted vegetation." }, { "id": 6188, "name": "snow line", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A boundary between snow-covered and snow-free terrain under a specified spatial and temporal definition. Seasonal, end-of-melt-season and persistent snow lines answer different questions." } ] }, { "task": "B19", "split": "development", "origin": "prospective-ai-task", "domain": "biology", "question": "I am evaluating a restored meadow after a survey reports twice as many plant species as before. Most new plants occupy very little area, one introduced grass dominates, and pollinator visits have fallen. Decide what can reasonably be called a restoration success. Distinguish richness, relative abundance, and ecological function, and identify the sampling checks and management options needed before recommending further planting.", "requirements": [ "Compare species richness with abundance-sensitive diversity.", "Assess functional evidence without treating pollinator counts as a complete ecosystem measure.", "Check comparability and detectability in the before-and-after surveys.", "Recommend management choices conditional on the evidence." ], "poolSize": 16, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3110, 2918, 3103, 5845, 3109 ], "ndcg5": 0.5586839730714311, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 1, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.5, "embeddingRedundancy": 0.529149941389808, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3110, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 2918, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 3103, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 5845, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3109, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3110, 2918, 3103, 5845, 3109 ], "ndcg5": 0.5586839730714311, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 1, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.5, "embeddingRedundancy": 0.529149941389808, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3110, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 2918, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 3103, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 5845, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3109, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 4563, 3087, 3110, 3117, 4600 ], "ndcg5": 0.7092603655638764, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.2, "embeddingRedundancy": 0.16138756049042893, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 4563, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 3087, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0 ] }, { "id": 3110, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3117, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 4600, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 0, 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 2 }, "oracle": { "top5": [ 3087, 3104, 3106, 3110, 3117 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 6, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.36177359460423575, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3087, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 3104, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 3106, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3110, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 3117, "agreedObligations": [ 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 0, 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 2 } }, "oracleEnumeration": { "subsetsEnumerated": 4368, "coverageMaximizingSubsets": 944, "coveredObligations": 4 }, "poolItems": [ { "id": 225, "name": "return and restoration after exile", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The hope and narrated movement of a dispersed people toward restored land, community, and worship. Return includes rebuilding material and social life, and it need not imply that loss is erased or that all displaced people return together." }, { "id": 1054, "name": "guarantees supported by domestic implementing law", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The memorandum distinguishes political assurances from the binding guarantees it seeks by linking a registered international instrument to domestic implementing legislation in each guarantor state. The intended bridge connects a diplomatic promise with the legal and administrative means for carrying it out." }, { "id": 2903, "name": "integrated pest management", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "Coordinating biological, cultural, physical and, where appropriate, chemical measures using knowledge of pests and local conditions. It seeks effective control while limiting unnecessary intervention and associated harm, rather than relying on a single routine treatment." }, { "id": 2918, "name": "rewilding", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 3 ], "definition": "An approach to ecological restoration that seeks to restore ecological processes and reduce dependence on continuing human management. Projects differ in their reference conditions, species interventions and openness to new ecological arrangements." }, { "id": 3087, "name": "species richness", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 2 ], "definition": "The number of species represented in a specified area, sample or assemblage. Its interpretation depends on sampling effort, taxonomic decisions and boundaries, and does not by itself describe how individuals are distributed among species." }, { "id": 3103, "name": "ecological succession", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "Change in the composition and structure of an ecological community through time, including colonization of new habitat and change following disturbance. Patterns can recur without requiring every site to follow one inevitable sequence toward a fixed final community." }, { "id": 3104, "name": "keystone species", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "A species whose effect on a community is disproportionately large relative to its abundance. The designation is a claim about ecological influence in a particular system, not simply a label for a large, rare or culturally important species." }, { "id": 3106, "name": "invasive species", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "An introduced species that establishes and spreads with harmful effects in a receiving environment. Being non-native alone does not establish invasiveness; ecological setting, spread and the nature of harm must be considered." }, { "id": 3109, "name": "conservation biology", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "An interdisciplinary field concerned with understanding and limiting biodiversity loss, combining ecological and evolutionary evidence with the social conditions of protection and resource use. Its goals involve values as well as empirical claims." }, { "id": 3110, "name": "ecological restoration", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "Assisting the recovery of an ecosystem that has been damaged, degraded or destroyed through interventions guided by ecological understanding and explicit goals. Recovery need not mean exact reproduction of one historical condition." }, { "id": 3117, "name": "ecosystem", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "Organisms and their physical environment considered as an interacting system, connected by transfers of energy and cycling of materials. Its boundary and scale depend on the question being studied; it is more than a list of species sharing a place." }, { "id": 4563, "name": "design-based survey inference", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "Inference about a target population based on the probabilities specified by a survey sampling design. The design supplies a sampling distribution but does not eliminate measurement, nonresponse or coverage bias." }, { "id": 4600, "name": "loss function", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "A declared function assigning a numerical penalty to an action or estimate in a specified state. Optimization or statistical decision rules compare options relative to this penalty." }, { "id": 5774, "name": "island biogeography equilibrium model", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A model relating species richness in isolated habitat to immigration and local extinction, with area and isolation affecting these rates and the expected richness balance." }, { "id": 5783, "name": "plant–soil feedback", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 3 ], "definition": "A process in which plants change soil biological or physical-chemical conditions, and those changes subsequently alter plant performance, positively or negatively." }, { "id": 5845, "name": "conservation agriculture", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A farming approach organized around limited soil disturbance, sustained soil cover and diversification of cultivated species, aiming to maintain production while protecting soil processes." } ] }, { "task": "B20", "split": "final", "origin": "prospective-ai-task", "domain": "biology", "question": "I advise a reserve where a reintroduced herbivore is increasing rapidly. Vegetation damage is concentrated near water, rainfall fluctuates, and predators have not returned. The board must choose between moving animals, restoring dispersal routes, or waiting. Explain how density dependence, uneven resource distribution, and variable carrying capacity change the decision, and propose thresholds that protect the habitat without treating one recent count as a reliable trend.", "requirements": [ "Distinguish reserve-wide abundance from local grazing pressure.", "Explain how rainfall and resource distribution affect population limits.", "Compare the proposed responses and their ecological consequences.", "Specify monitoring and action thresholds with uncertainty acknowledged." ], "poolSize": 14, "poolAttainableObligations": [ 1, 2, 3 ], "poolMissingObligations": [ 0 ], "metrics": { "dense": { "top5": [ 3103, 3125, 2918, 1824, 75 ], "ndcg5": 0.4698115003932712, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.5, "embeddingRedundancy": 0.5251334853456568, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3103, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3125, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 2918, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 1824, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 75, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3103, 3125, 2918, 1824, 75 ], "ndcg5": 0.4698115003932712, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.5, "embeddingRedundancy": 0.5251334853456568, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 3103, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3125, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 2918, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 1824, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 75, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3125, 84, 3099, 3103, 1639 ], "ndcg5": 0.49116748177133657, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.26815775989671864, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3125, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 84, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3099, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 3103, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1639, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 1 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 }, "oracle": { "top5": [ 3099, 4387, 1824, 2918, 3125 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.625, "embeddingRedundancy": 0.434154999746332, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3099, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 4387, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 1824, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 2918, "agreedObligations": [ 2 ], "newObligationsAtRank": [] }, { "id": 3125, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 1 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 2002, "coverageMaximizingSubsets": 754, "coveredObligations": 3 }, "poolItems": [ { "id": 75, "name": "closed-loop habitat", "grade": 1.5, "judgeA": 2, "judgeB": 1, "agreedObligations": [], "definition": "A habitat designed by treating air, water, food, waste, energy, organisms, and maintenance as coupled cycles whose inputs, outputs, losses, and dependencies are explicit." }, { "id": 84, "name": "reserve before scale", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "Maintaining sufficient reserve capacity before increasing exposure, complexity, or operating scale so growth does not consume the system's ability to survive error." }, { "id": 1208, "name": "uneven development of judgment", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The coexistence of unusual skill or insight in one domain with ordinary prejudice, vanity or poor judgment in another. Excellence in a field does not automatically transfer to every practical or interpersonal decision." }, { "id": 1513, "name": "withdrawal at completed accomplishment", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Ceasing to press an advantage once the work has been completed, rather than using success as a reason for indefinite display, accumulation or control." }, { "id": 1639, "name": "marriage without power to protect a partner", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A legally or socially recognized intimate bond that gives one partner no effective capacity to protect the other from a third party’s coercive claim." }, { "id": 1824, "name": "habitat alteration changing relations among survivors", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "Changing a habitat can remove some occupants while altering the relationships among those that remain. Counting losses alone does not capture the reorganization of the continuing community." }, { "id": 2918, "name": "rewilding", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2 ], "definition": "An approach to ecological restoration that seeks to restore ecological processes and reduce dependence on continuing human management. Projects differ in their reference conditions, species interventions and openness to new ecological arrangements." }, { "id": 3099, "name": "carrying capacity", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "The upper population level an environment can sustain under stated resource, waste and time conditions. In simple population models it can be an equilibrium parameter; in changing environments it is not necessarily constant or directly observable." }, { "id": 3103, "name": "ecological succession", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "Change in the composition and structure of an ecological community through time, including colonization of new habitat and change following disturbance. Patterns can recur without requiring every site to follow one inevitable sequence toward a fixed final community." }, { "id": 3109, "name": "conservation biology", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An interdisciplinary field concerned with understanding and limiting biodiversity loss, combining ecological and evolutionary evidence with the social conditions of protection and resource use. Its goals involve values as well as empirical claims." }, { "id": 3125, "name": "ecological resilience", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "An ecosystem's capacity to absorb disturbance and reorganize while retaining characteristic functions, structure and feedbacks. Resilience need not mean returning to an identical prior state, and its assessment depends on the functions and boundaries chosen." }, { "id": 4387, "name": "trophic cascade", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2 ], "definition": "An indirect ecological effect transmitted across trophic levels, for example when changing predators changes herbivore pressure on producers. Direction and strength depend on the food web and conditions; restoring one species need not guarantee a desired ecosystem outcome." }, { "id": 6201, "name": "debris flow", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A rapidly moving mixture of sediment, rock fragments and water whose concentrated solid content and pore-pressure behavior shape its motion. It differs from ordinary clear-water flooding and can entrain additional material." }, { "id": 6206, "name": "slug test", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A groundwater test in which water level is rapidly displaced in a well and its recovery is monitored. It commonly probes conditions near that well rather than the same spatial volume as a prolonged pumping test." } ] }, { "task": "B21", "split": "final", "origin": "prospective-ai-task", "domain": "economics", "question": "I advise a city that can fund either a universal home-insulation rebate or a larger rebate limited to low-income households. Many tenants do not control building improvements, and some owners would renovate without support. Recommend a design that considers additional energy savings, landlord-tenant incentives, rebound in energy use, and distribution of benefits. Specify what should be measured in a pilot before expanding it.", "requirements": [ "Compare additionality and deadweight spending across designs.", "Account for split incentives between owners and tenants.", "Consider rebound and net energy savings.", "Recommend a design and pilot evidence addressing both effectiveness and distribution." ], "poolSize": 14, "poolAttainableObligations": [ 1, 2, 3 ], "poolMissingObligations": [ 0 ], "metrics": { "dense": { "top5": [ 2162, 4377, 1599, 917, 2148 ], "ndcg5": 0.5761076079923059, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 1, 3 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.3274731907735587, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 2162, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 4377, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1599, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 917, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2148, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 2162, 4377, 1599, 917, 2148 ], "ndcg5": 0.5761076079923059, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 1, 3 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.3274731907735587, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 2162, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 4377, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1599, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 917, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2148, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 2162, 2878, 4384, 5106, 4377 ], "ndcg5": 0.725070381813123, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 3, "labelRedundancy": 0.0, "embeddingRedundancy": 0.10610117579239724, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 2162, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 2878, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4384, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 5106, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4377, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 }, "oracle": { "top5": [ 2162, 3219, 4384, 1599, 4377 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 1, 2, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.24809132352439334, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 2162, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] }, { "id": 3219, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 4384, "agreedObligations": [ 2 ], "newObligationsAtRank": [ 2 ] }, { "id": 1599, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4377, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 2002, "coverageMaximizingSubsets": 220, "coveredObligations": 3 }, "poolItems": [ { "id": 815, "name": "the exiles called to seek their captor city’s welfare", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Jeremiah’s letter instructs exiles to build, plant, form households, and seek the welfare of the city where they have been sent, linking their own peace with its peace. Continuing life in displacement becomes a present responsibility without canceling the stated hope of return." }, { "id": 917, "name": "protected voluntary return with reintegration support", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The draft proposes connecting reintegration funding, border hotlines and independent flow monitoring to orderly, voluntary return and protection against forced return. Housing, work and essential services are intended to support people’s choices rather than turn displacement into a reward mechanism." }, { "id": 1199, "name": "earned income without control", "grade": 0.5, "judgeA": 0, "judgeB": 1, "agreedObligations": [], "definition": "An arrangement in which a person produces wages or revenue but another claims the proceeds and determines what, if anything, the earner may retain. Earning ability is distinguished from control over its product." }, { "id": 1599, "name": "land rent distinguished from payment for improvements", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "The distinction between payment for a site’s productive advantages and payment for buildings, drainage or other invested improvements bundled into the same annual charge." }, { "id": 1900, "name": "analysis of a work distinguished from experiencing its force", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Keller distinguishes knowing a work’s parts and explanations from enjoying its living effect. Interpretation can restore access to that effect, while excessive commentary can obscure it; the distinction is not a rejection of careful knowledge." }, { "id": 2148, "name": "retaining private possession while redirecting its yield", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An institutional design that leaves possession and transfer forms in place while redirecting a specified economic return to public use. George proposes this for land rent; formal continuity does not mean that the distribution of benefits remains unchanged." }, { "id": 2162, "name": "improvement gains absorbed through later rent increases", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "A tenant’s higher output or favorable prices are followed by increased rent, while adverse changes receive less corresponding relief. Anticipated gains from effort can therefore pass to another party instead of improving the tenant’s position." }, { "id": 2878, "name": "universal design", "grade": 0.5, "judgeA": 0, "judgeB": 1, "agreedObligations": [], "definition": "Design intended to make products, environments and services usable by as many people as possible across differing abilities and circumstances. It anticipates common barriers, while recognizing that conflicting needs can still require adaptation or alternative arrangements." }, { "id": 3219, "name": "just transition", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "An approach to environmental and economic change that seeks to protect participation, livelihoods and fair opportunities for affected people and communities. It treats the distribution of transition costs and benefits as part of the transition's design." }, { "id": 4377, "name": "urban planning", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "The development of plans for land use, settlements, infrastructure and access, involving physical, social and environmental conditions. A plan’s technical coherence does not establish resident consent, equitable effects or authority to implement it." }, { "id": 4384, "name": "resource-efficiency rebound", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 2 ], "definition": "A reduction of expected resource savings when efficiency improvements induce behavioral or systemic changes, such as greater use. Its size depends on the setting; an efficiency gain alone establishes neither a guaranteed absolute saving nor inevitable backfire." }, { "id": 5106, "name": "GDP per capita", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Gross domestic product divided by the population of the measured economy. It is an average production indicator and can be price-adjusted for comparisons, but does not describe the distribution of income or a typical person's actual consumption." }, { "id": 5335, "name": "precarity", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "Insecurity in the conditions of life or work, including limited predictability, reliable income, protection or control over future arrangements. It can overlap with poverty without being identical to a low income measured at one moment." }, { "id": 6154, "name": "material hardness", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Resistance measured by a specified local indentation, scratching or rebound procedure. Different hardness tests probe different responses and require a named scale; hardness is not a universal substitute for toughness or tensile strength." } ] }, { "task": "B22", "split": "final", "origin": "prospective-ai-task", "domain": "economics", "question": "I manage a water utility with high fixed costs and increasingly frequent dry seasons. A higher per-unit price could reduce use, but low-income households may have little discretionary consumption and falling sales could undermine maintenance funding. Compare increasing-block tariffs, a uniform scarcity charge with rebates, and fixed fees. Recommend a structure that balances conservation incentives, revenue stability, and affordability under uncertain demand.", "requirements": [ "Distinguish fixed-cost recovery from marginal scarcity incentives.", "Compare the incidence and incentives of the three tariff designs.", "Address uncertain demand and the utility's revenue requirement.", "Recommend a design with an explicit affordability mechanism." ], "poolSize": 15, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 4379, 3876, 1596, 2896, 5092 ], "ndcg5": 0.4485515486456346, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.31231776692351054, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 4379, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] }, { "id": 3876, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 1596, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2896, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5092, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 4379, 3876, 1596, 2896, 5092 ], "ndcg5": 0.4485515486456346, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 4, "labelRedundancy": 0.25, "embeddingRedundancy": 0.31231776692351054, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 4379, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] }, { "id": 3876, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 1596, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2896, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5092, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 1, 3 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 1596, 3876, 4379, 3922, 740 ], "ndcg5": 0.3300727127739191, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 1 ], "agreedAssignments": 3, "labelRedundancy": 0.3333333333333333, "embeddingRedundancy": 0.24241967772306347, "documentsWithoutAgreedObligations": 2, "increments": [ { "id": 1596, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3876, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] }, { "id": 4379, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 3922, "agreedObligations": [ 1 ], "newObligationsAtRank": [ 1 ] }, { "id": 740, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [ 3 ], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 1, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 1609, 1675, 3929, 5085, 4379 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 9, "labelRedundancy": 0.5555555555555556, "embeddingRedundancy": 0.20345213558234598, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 1609, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 1675, "agreedObligations": [ 0, 2 ], "newObligationsAtRank": [] }, { "id": 3929, "agreedObligations": [ 1, 2 ], "newObligationsAtRank": [ 1 ] }, { "id": 5085, "agreedObligations": [ 1, 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 4379, "agreedObligations": [ 0 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 3003, "coverageMaximizingSubsets": 1149, "coveredObligations": 4 }, "poolItems": [ { "id": 740, "name": "Surah 96: Learning against self-sufficiency and obstruction", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A reading frame beginning with recitation, creation, and teaching through the pen, then confronting the insolence of imagined self-sufficiency. An attempt to obstruct a praying servant turns the contrast into a conflict over worship, answered by a summons to draw near." }, { "id": 1596, "name": "scarcity of an unreproducible good", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A supply limitation that additional ordinary productive labor cannot remove, so the good’s exchange value cannot be explained by the same conditions used for reproducible goods." }, { "id": 1609, "name": "tax rate distinguished from tax receipts", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 2 ], "definition": "The distinction between the charge imposed per taxable unit and the total revenue collected, allowing changes in participation or consumption to make a higher rate yield less revenue." }, { "id": 1675, "name": "equal per-person funding below a viable service threshold", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 2 ], "definition": "A formally proportional allocation that leaves a small or dispersed group unable to obtain a functioning service because essential fixed costs are not covered." }, { "id": 2886, "name": "water footprint", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Freshwater use associated with producing or consuming specified goods and services, including direct and supply-chain use under a declared accounting method. Its meaning depends on where and when water is used and how consumption or pollution is represented." }, { "id": 2896, "name": "renewable energy", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Energy obtained from natural flows or resources replenished on human timescales. Renewability concerns replenishment, whereas emissions, habitat effects, reliability and sustainable extraction are separate properties requiring assessment." }, { "id": 2911, "name": "water reuse", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Using water again after a prior use, with treatment and monitoring appropriate to its next purpose. Reuse can reduce demand on freshwater sources, but irrigation, industrial uses and drinking water impose different quality and exposure requirements." }, { "id": 3876, "name": "commons overuse dilemma", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "A pattern in which individual benefits from using a shared, subtractable resource can encourage cumulative extraction that damages the common resource. Such a dilemma depends on incentives and governance; it is not an unavoidable fate of every collectively managed commons." }, { "id": 3922, "name": "marginal utility", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1 ], "definition": "The change in represented utility associated with a small additional consumption or allocation from a specified baseline. The sign and size concern that increment, not the total utility of all units already held." }, { "id": 3929, "name": "price elasticity of demand", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 2 ], "definition": "The proportional change in quantity demanded associated with a proportional price change, holding other modeled influences fixed. It measures relative responsiveness rather than a curve's unscaled slope; the sign and whether magnitude alone is reported must be stated." }, { "id": 4379, "name": "water scarcity", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 0 ], "definition": "Insufficient accessible freshwater of suitable quality to meet specified demands in a place and time. Physical shortage and lack of capacity or infrastructure are different causes; a global quantity alone cannot describe local access or ecological requirements." }, { "id": 5082, "name": "normal good", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 2 ], "definition": "A good for which demand rises when the consumer's income rises, other relevant conditions held constant. Normality is an income-response classification that can vary by consumer and income range, not a judgment about quality or socially appropriate consumption." }, { "id": 5085, "name": "income effect", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "The part of a change in consumption attributable to changed purchasing power, including the purchasing-power consequence of a price change. It is separated analytically from substitution due to changed relative prices." }, { "id": 5092, "name": "rationing", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 1, 3 ], "definition": "Allocation of limited goods, services or access through rules controlling who receives how much or when. Prices, queues, quotas and coupons are different mechanisms, each with consequences for distribution and access rather than one automatically fair solution." }, { "id": 5106, "name": "GDP per capita", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Gross domestic product divided by the population of the measured economy. It is an average production indicator and can be price-adjusted for comparisons, but does not describe the distribution of income or a typical person's actual consumption." } ] }, { "task": "B23", "split": "final", "origin": "prospective-ai-task", "domain": "economics", "question": "I help a worker cooperative choose between higher guaranteed wages and a profit-sharing bonus. Individual effort is difficult to observe, sales are volatile, and employees differ in how much income risk they can bear. Compare incentive effects, free riding, and risk allocation, then recommend a compensation rule that remains credible when profits fall and does not reward accounting changes more than productive work.", "requirements": [ "Compare incentive and free-riding effects of the alternatives.", "Explain how income risk is distributed across workers and the organization.", "Address measurement and accounting manipulation.", "Propose a feasible rule for both profitable and weak years." ], "poolSize": 15, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 944, 3967, 2473, 1034, 5049 ], "ndcg5": 0.6494782413548044, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.5714285714285714, "embeddingRedundancy": 0.4538325111043502, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 944, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3967, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 2473, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 1034, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 5049, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 944, 3967, 2473, 1034, 5049 ], "ndcg5": 0.6494782413548044, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.5714285714285714, "embeddingRedundancy": 0.4538325111043502, "documentsWithoutAgreedObligations": 1, "increments": [ { "id": 944, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3967, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [ 0, 2 ] }, { "id": 2473, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 1034, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 5049, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 2423, 5049, 299, 1646, 5118 ], "ndcg5": 0.27874552662851343, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 2, 3 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.1703069654219786, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 2423, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 5049, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 299, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1646, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 5118, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [ 0 ], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 1, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3938, 3967, 5236, 2473, 944 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.5, "embeddingRedundancy": 0.42498434921093553, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3938, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 3967, "agreedObligations": [ 0, 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 5236, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 2473, "agreedObligations": [ 3 ], "newObligationsAtRank": [] }, { "id": 944, "agreedObligations": [ 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 1 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 } }, "oracleEnumeration": { "subsetsEnumerated": 3003, "coverageMaximizingSubsets": 807, "coveredObligations": 4 }, "poolItems": [ { "id": 299, "name": "accountability for entrusted resources", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Matthew's talents story connects an entrusted resource with later accounting, contrasting productive service with fearful nonuse. Reward and loss are features of the master's judgment within the parable." }, { "id": 939, "name": "mediator exit as a cooperation incentive", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A draft bargaining mechanism in which a facilitator's credible ability to suspend mediation raises the cost of continued non-cooperation. The threatened withdrawal concerns facilitation services and does not itself authorize withholding humanitarian access or individual protections." }, { "id": 944, "name": "adjusting incentives to shorter decision horizons", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "A draft response to reduced willingness to wait: move visible cooperation benefits earlier and make proposed breach costs more immediate, while rechecking participation conditions. The adjustment changes the model's reward schedule; it does not demonstrate the actors' actual discount factors or justify overriding protected needs." }, { "id": 1034, "name": "Draft 11 frame: incentive analysis", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0, 3 ], "definition": "A proposed incentive model relating repeated interaction, detection, losses and future value to compliance. Its equations and illustrative parameters are inspectable analytical choices, distinct from measured preferences, validated predictions or evidence that every participant faces the same payoff structure." }, { "id": 1646, "name": "opposition to a rule distinguished from accepting shelter’s risk", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The distinction between rejecting a coercive rule in principle and being willing to incur the personal legal or social risk of protecting someone threatened by it." }, { "id": 1781, "name": "volitional effort as holding a difficult consideration", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "James proposes that effortful willing centrally involves keeping a difficult thought present against competing tendencies. This is a theory of the relation between attention and action, not a universal prescription to overcome illness by willpower." }, { "id": 2121, "name": "economic dependence shaping reception of a warning", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "A situation in which a community’s reliance on an institution’s income makes evidence of that institution’s danger difficult to acknowledge. The costs of remedy and lost custom become entangled with judgments about whether the warning is credible." }, { "id": 2326, "name": "continued collective risk limited by those who bear it", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 1, 3 ], "definition": "Leadership of a hazardous undertaking does not make initial participation unlimited consent to renewed danger. Participants who bear the risk can demand a change of course, and a leader may act on that limit even while emotionally regarding retreat as failure or injustice." }, { "id": 2423, "name": "a small net effect sustained by large opposing contributions", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "A modest resultant can conceal much larger opposing flows, forces or expenditures. Assessing feasibility requires accounting for those gross contributions and their losses or side effects, not only the small difference left after cancellation." }, { "id": 2473, "name": "a trust-indexed incentive with a funding ceiling", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "A proposed mechanism increases compensating incentives when a breach reduces confidence in future performance. Its recovery claim is bounded by available reserves and credible implementation; if the required compensation exceeds those limits, increasing a modeled coefficient is not an available remedy." }, { "id": 3938, "name": "moral hazard", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "An incentive to change behavior when another party bears some of the consequences, particularly when actions are difficult to observe or constrain. The problem concerns altered incentives after risk-sharing or delegation, not a necessary judgment about an actor's moral character." }, { "id": 3967, "name": "incentive compatibility", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 2, 3 ], "definition": "A property of a mechanism under which truthful reporting or prescribed participation is a best response for each participant under the stated strategic conditions. Dominant-strategy compatibility holds regardless of others' choices; Bayesian compatibility depends on the modeled beliefs and equilibrium." }, { "id": 5049, "name": "division of labor", "grade": 1.5, "judgeA": 1, "judgeB": 2, "agreedObligations": [], "definition": "Separation of an activity into tasks performed by different people or units, allowing specialization while increasing dependence on coordination and exchange. Productivity effects depend on the arrangement and do not establish that the resulting distribution of power or rewards is fair." }, { "id": 5118, "name": "automatic fiscal stabilizer", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A budget feature, such as income-responsive taxes or benefits, that changes with economic conditions without a new discretionary policy decision each time. The resulting response can dampen fluctuations, though its strength depends on program design and coverage." }, { "id": 5236, "name": "positive goal interdependence", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0 ], "definition": "A relation in which participants perceive that their own goal attainment is linked to others' success. In cooperative learning this supports helping one another, but a shared incentive does not by itself establish equal influence or guarantee cooperation." } ] }, { "task": "B24", "split": "development", "origin": "prospective-ai-task", "domain": "economics", "question": "I advise a regional employment office considering training vouchers for an industry reporting vacancies. Employers also offer low wages and have high turnover, while some applicants would pay for training themselves. Decide whether a pilot is justified and how to evaluate it. Distinguish a skill shortage from recruitment problems, private from social returns, and genuine employment gains from displacement of other workers.", "requirements": [ "Compare skill shortages with wage or working-condition explanations.", "Distinguish additional training from training that would happen anyway.", "Assess private benefits, social benefits, and worker displacement.", "Recommend a pilot decision and evaluation design with meaningful outcomes." ], "poolSize": 15, "poolAttainableObligations": [ 0, 2, 3 ], "poolMissingObligations": [ 1 ], "metrics": { "dense": { "top5": [ 2309, 1972, 1129, 1388, 5053 ], "ndcg5": 0.40038843218805914, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 3, "labelRedundancy": 0.0, "embeddingRedundancy": 0.2432688638740415, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 2309, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1972, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1129, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1388, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 5053, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 2309, 1972, 1129, 1388, 5053 ], "ndcg5": 0.40038843218805914, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 3, "labelRedundancy": 0.0, "embeddingRedundancy": 0.2432688638740415, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 2309, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1972, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1129, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1388, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 5053, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 2058, 2163, 1386, 2309, 4401 ], "ndcg5": 0.3789074958111966, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 3, "labelRedundancy": 0.0, "embeddingRedundancy": 0.11967151249683135, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 2058, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 2163, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] }, { "id": 1386, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2309, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4401, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 5053, 5054, 1388, 2058, 2163 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 2, 3 ], "agreedAssignments": 7, "labelRedundancy": 0.5714285714285714, "embeddingRedundancy": 0.26562225298863795, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 5053, "agreedObligations": [ 0 ], "newObligationsAtRank": [ 0 ] }, { "id": 5054, "agreedObligations": [ 0 ], "newObligationsAtRank": [] }, { "id": 1388, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2, 3 ] }, { "id": 2058, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [] }, { "id": 2163, "agreedObligations": [ 0 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 3003, "coverageMaximizingSubsets": 1176, "coveredObligations": 3 }, "poolItems": [ { "id": 211, "name": "anointing for office", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The use of anointing to mark a person for a designated sacred or royal role. The sign identifies consecration or commission, while the wider narratives still distinguish office, personal conduct, succession, and the loss of favor." }, { "id": 1129, "name": "notice and an end date for a temporary foreign training presence", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The draft distinguishes rotational training from a permanent foreign footprint and requires notice, limits and a published end date for specified longer missions. Permission, notification and compliance verification are separate stages; a temporary label does not by itself satisfy the conditions." }, { "id": 1386, "name": "a private pilot becoming a common standard", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A path in which an initially voluntary experiment becomes a subject of public discussion, shared adoption and general protection. The first initiative and the later authority of a common rule are different stages, each requiring its own justification." }, { "id": 1388, "name": "education for those who remain in the work", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "Education that enriches understanding and agency within necessary ordinary work is distinguished from education whose main success is moving a few people out of it. Occupational mobility for some does not meet every learning need of those who continue the activity." }, { "id": 1972, "name": "training withheld to prevent future competition", "grade": 1.5, "judgeA": 2, "judgeB": 1, "agreedObligations": [], "definition": "Excluding a prospective learner because successful training could create a rival to those controlling admission. The exclusion protects an incumbent position rather than establishing the applicant’s incapacity." }, { "id": 2058, "name": "training success measured beside imposed injury", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 2, 3 ], "definition": "Assessment of a training practice that counts the injury, deprivation and loss caused in producing the desired performance. The trainer’s claim to mastery does not settle whether the transformation benefits the trained beings." }, { "id": 2163, "name": "labor mobility restricted through barriers to recruitment", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "Restrictions on recruiting workers for outside employment can reduce both movement and knowledge of alternatives. A person’s remaining in one labor market does not by itself establish a free preference for its available terms." }, { "id": 2309, "name": "vocation as continuing discovery across multiple callings", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A person has several connected callings, and choosing a specialized occupation outlines a field for further discovery rather than fixing a final destiny. Vocational education should sustain revision of capacities, aims and methods as experience develops." }, { "id": 4120, "name": "internal control", "grade": 0.5, "judgeA": 0, "judgeB": 1, "agreedObligations": [], "definition": "Processes used by an organization to provide reasonable assurance about operations, reporting and compliance objectives. Controls may prevent, detect or respond to problems, but their design and operation cannot guarantee objectives whose achievement also depends on conditions outside their scope." }, { "id": 4352, "name": "labour economics", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "The study of labour as an input to production and of workers, employers and institutions in labour markets. Wages, participation, bargaining and discrimination are different questions; describing a market does not establish that its outcomes are fair." }, { "id": 4401, "name": "outgassing", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Release of gas or volatile material from substances through processes such as desorption, evaporation or slow reactions. In vacuum systems the released material can contaminate surfaces; pressure, temperature and material history affect the process." }, { "id": 5053, "name": "reservation wage", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0 ], "definition": "The lowest wage at which a person would accept a particular job under specified conditions and available alternatives. It may change with household needs, risks and job characteristics, so it is not simply a fixed measure of willingness to work." }, { "id": 5054, "name": "efficiency wage", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0 ], "definition": "A wage above a modeled market-clearing level that an employer may pay to improve performance, recruitment or retention. Efficiency-wage theories explain possible incentives for this choice without implying that higher pay always produces a proportional productivity increase." }, { "id": 5056, "name": "collective bargaining", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Negotiation between workers' representatives and employers about pay, conditions or related employment rules. It may occur at workplace, sectoral or other levels; the process is distinct from an agreement reached and from its subsequent implementation." }, { "id": 5057, "name": "collective labor agreement", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An agreement resulting from negotiations between workers' representatives and employers, setting specified employment terms or procedures. Its scope, binding effect and implementation depend on the agreement and relevant legal framework." } ] } ], "preselectedCases": { "A21": { "task": "A21", "split": "final", "origin": "prospective-ai-task", "domain": "statistics", "question": "You are reviewing a team's randomized website experiment. They compare two designs on three outcome measures, inspect results every day, and want to stop as soon as any conventional significance threshold is crossed. Explain how repeated looks and multiple outcomes differ, separate statistical evidence from practical effect size, and recommend a decision procedure that preserves an interpretable error guarantee.", "requirements": [ "Distinguish multiplicity across outcomes from repeated sequential looks.", "Explain how the proposed stopping rule affects an ordinary fixed-sample analysis.", "Incorporate effect size and uncertainty into the decision goal.", "Recommend a prespecified procedure with clear error control and stopping rules." ], "poolSize": 15, "poolAttainableObligations": [ 0, 1, 2, 3 ], "poolMissingObligations": [], "metrics": { "dense": { "top5": [ 3449, 3472, 3475, 3478, 4699 ], "ndcg5": 0.4710468685170683, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 3 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.4316145109250443, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 3449, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 3472, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3475, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3478, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4699, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3472, 3449, 3475, 3478, 4699 ], "ndcg5": 0.348585852916874, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 3 ], "agreedAssignments": 2, "labelRedundancy": 0.0, "embeddingRedundancy": 0.4316145109250443, "documentsWithoutAgreedObligations": 4, "increments": [ { "id": 3472, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3449, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0, 3 ] }, { "id": 3475, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3478, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4699, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 4684, 3475, 3449, 147, 4288 ], "ndcg5": 0.6155580477282292, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 5, "labelRedundancy": 0.4, "embeddingRedundancy": 0.12382569272466142, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 4684, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 0, 1, 3 ] }, { "id": 3475, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3449, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [] }, { "id": 147, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 4288, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 1 ], "lostCountVsDense": 0, "gainedCountVsDense": 1 }, "oracle": { "top5": [ 3429, 3449, 4684, 3428, 3426 ], "ndcg5": 1.0, "agreedObligationCoverage5": 1.0, "coveredObligations": [ 0, 1, 2, 3 ], "agreedAssignments": 9, "labelRedundancy": 0.5555555555555556, "embeddingRedundancy": 0.3397609153127689, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3429, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3449, "agreedObligations": [ 0, 3 ], "newObligationsAtRank": [ 0 ] }, { "id": 4684, "agreedObligations": [ 0, 1, 3 ], "newObligationsAtRank": [ 1 ] }, { "id": 3428, "agreedObligations": [ 2, 3 ], "newObligationsAtRank": [ 2 ] }, { "id": 3426, "agreedObligations": [ 3 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [ 1, 2 ], "lostCountVsDense": 0, "gainedCountVsDense": 2 } }, "oracleEnumeration": { "subsetsEnumerated": 3003, "coverageMaximizingSubsets": 286, "coveredObligations": 4 }, "poolItems": [ { "id": 147, "name": "the binding of Isaac", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The narrative of Abraham commanded to offer his son, brought to the point of sacrifice, and then stopped, with a ram supplied instead. It concentrates promise, obedience, danger, and substitution in a scene whose moral and theological implications remain contested." }, { "id": 2750, "name": "eventual consistency", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A replication guarantee that, if further updates stop and the required communication and recovery conditions hold, replicas converge to a common resolved state. It permits temporary disagreement and does not by itself bound how soon convergence occurs or what intermediate reads observe." }, { "id": 3426, "name": "statistical hypothesis test", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "A decision procedure comparing observations with expectations under a specified statistical hypothesis. Rejection or non-rejection depends on assumptions and a decision rule; failing to reject does not establish that the hypothesis is true." }, { "id": 3428, "name": "statistical power", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 2, 3 ], "definition": "The probability that a test rejects its null hypothesis for a specified alternative when that alternative holds. Power depends on effect, design, sample size, noise and decision threshold; it is not a property of sample size alone." }, { "id": 3429, "name": "statistical decision errors", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "A type I error rejects a true null hypothesis; a type II error fails to reject it under a specified alternative. Their probabilities belong to the chosen hypotheses and decision procedure, rather than a general measure of factual correctness." }, { "id": 3449, "name": "multiple comparisons problem", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 3 ], "definition": "The increased opportunity for misleading findings when many statistical comparisons are made and interpreted selectively. Error control depends on the family of questions and procedure, not only the threshold for each individual test." }, { "id": 3450, "name": "meta-analysis", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Statistical synthesis of results from multiple studies addressing related questions. Validity depends on study compatibility, bias, uncertainty and selection; combining estimates does not automatically remove weaknesses shared by the studies." }, { "id": 3460, "name": "random measurement error", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A component of measurement error varying unpredictably between repeated measurements under stated conditions. Statistical treatment can characterize its variability; randomness does not imply that the complete measurement is unbiased." }, { "id": 3472, "name": "experiment", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An inquiry that deliberately controls or changes conditions to examine resulting observations against a question or hypothesis. Its inferential strength depends on design, comparison and measurement, not merely on intervening." }, { "id": 3475, "name": "randomized controlled trial", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "An experiment assigning eligible units to interventions by a random mechanism and comparing outcomes. Randomization limits certain confounding in expectation, but attrition, nonadherence, measurement and limited scope can still affect interpretation." }, { "id": 3478, "name": "quasi-experiment", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An intervention study without full random assignment that uses design or analysis to support a causal comparison. Its identifying assumptions require explicit scrutiny because treatment groups may differ for other reasons." }, { "id": 4189, "name": "hierarchy of evidence", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A heuristic ordering of study designs or evidence types by their expected strength for a specified question, often by susceptibility to bias. The ranking depends on the question, outcome and execution; a high-ranked design is not a guarantee that a particular result is sound." }, { "id": 4288, "name": "Buddhist two-truths doctrine", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A distinction between conventional and ultimate truth or reality in Buddhist teaching. Schools differ about how the two relate; distinguishing levels does not automatically license any contradiction or make conventional concerns disposable." }, { "id": 4684, "name": "sequential statistical analysis", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1, 3 ], "definition": "Evaluating accumulating data under a prespecified monitoring and stopping procedure, allowing sample size to depend on observations while accounting for the repeated decisions." }, { "id": 4699, "name": "experimental blocking", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Grouping experimental units by relevant nuisance variation and arranging treatment comparisons within those groups. The purpose is to separate the treatment comparison from known sources of variability." } ] }, "B18": { "task": "B18", "split": "final", "origin": "prospective-ai-task", "domain": "biology", "question": "I am a lake-restoration officer choosing between reducing nutrient inflow, removing a common fish, or combining the two. The lake is persistently cloudy, but fish surveys and nutrient measurements vary strongly by season. Explain how food-web interactions, external inputs, and delayed ecosystem responses could support different choices, and propose monitoring that can separate treatment effects from ordinary seasonal variation.", "requirements": [ "Compare the candidate interventions through plausible ecological pathways.", "Separate external nutrient inputs from effects within the food web.", "Address delayed responses or feedbacks.", "Design a comparison and monitoring schedule that handles seasonality." ], "poolSize": 16, "poolAttainableObligations": [ 0, 1, 3 ], "poolMissingObligations": [ 2 ], "metrics": { "dense": { "top5": [ 3111, 3090, 3120, 75, 3077 ], "ndcg5": 0.47697017530379393, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 3, "labelRedundancy": 0.0, "embeddingRedundancy": 0.36304879140716934, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3111, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3090, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 3120, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 75, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3077, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "typed": { "top5": [ 3111, 3090, 3120, 75, 3121 ], "ndcg5": 0.47697017530379393, "agreedObligationCoverage5": 0.5, "coveredObligations": [ 0, 1 ], "agreedAssignments": 4, "labelRedundancy": 0.5, "embeddingRedundancy": 0.4122163174148422, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3111, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3090, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 3120, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 75, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 3121, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [ 3 ], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 1, "gainedCountVsDense": 0 }, "coverage": { "top5": [ 3077, 3121, 3111, 2740, 1970 ], "ndcg5": 0.6038946136966565, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 3, "labelRedundancy": 0.0, "embeddingRedundancy": 0.12691770248564566, "documentsWithoutAgreedObligations": 3, "increments": [ { "id": 3077, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3121, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 3111, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 2740, "agreedObligations": [], "newObligationsAtRank": [] }, { "id": 1970, "agreedObligations": [], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 }, "oracle": { "top5": [ 3077, 3090, 3121, 4387, 3117 ], "ndcg5": 1.0, "agreedObligationCoverage5": 0.75, "coveredObligations": [ 0, 1, 3 ], "agreedAssignments": 8, "labelRedundancy": 0.625, "embeddingRedundancy": 0.3790019126314489, "documentsWithoutAgreedObligations": 0, "increments": [ { "id": 3077, "agreedObligations": [ 3 ], "newObligationsAtRank": [ 3 ] }, { "id": 3090, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [ 0, 1 ] }, { "id": 3121, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 4387, "agreedObligations": [ 0, 1 ], "newObligationsAtRank": [] }, { "id": 3117, "agreedObligations": [ 1 ], "newObligationsAtRank": [] } ], "uniqueObligationsLostVsDense": [], "uniqueObligationsGainedVsDense": [], "lostCountVsDense": 0, "gainedCountVsDense": 0 } }, "oracleEnumeration": { "subsetsEnumerated": 4368, "coverageMaximizingSubsets": 2155, "coveredObligations": 3 }, "poolItems": [ { "id": 75, "name": "closed-loop habitat", "grade": 1.5, "judgeA": 2, "judgeB": 1, "agreedObligations": [], "definition": "A habitat designed by treating air, water, food, waste, energy, organisms, and maintenance as coupled cycles whose inputs, outputs, losses, and dependencies are explicit." }, { "id": 1124, "name": "separate measures for infrastructure ecology and heritage harm", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "A draft monitoring agenda keeps power capacity loss, pollution, ecological damage and cultural-site damage as different kinds of impact. Each needs appropriate units, baselines and evidence; a shared dashboard must not make unlike measurements interchangeable or infer a common attribution from proximity." }, { "id": 1558, "name": "seasonal variation recorded in sediment layers", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 3 ], "definition": "A proposed relation between recurring changes in water volume and transport and alternating layers of coarser and finer deposited material." }, { "id": 1970, "name": "internal mutual support alongside external subjection", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "An institution may support its own members while collectively dominating outsiders. Internal reciprocity and the treatment of nonmembers therefore require separate evaluation." }, { "id": 2442, "name": "historical description, current allocation", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "The distinction between an earlier description retained as history and the validated registry governing present use. A linked amendment identifies the changed assignment and its reason without silently rewriting the earlier wording. Historical references remain tied to their original snapshots; preserving a statement does not renew an obsolete allocation." }, { "id": 2740, "name": "memoization", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "Reusing stored results when the same function inputs recur, avoiding repeated computation at the cost of storage. The stored result must remain valid for all relevant dependencies; hidden state or side effects can make apparently identical calls require different treatment." }, { "id": 3077, "name": "environmental monitoring", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 3 ], "definition": "Repeated or organized observation of environmental conditions using a design suited to a stated question. It can establish baselines and trends, while requiring attention to sampling, measurement and interpretation before change or its cause is inferred." }, { "id": 3090, "name": "nutrient cycle", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "Movement and transformation of biologically relevant matter among organisms and environmental reservoirs. Materials can be reused through linked processes; this cycling differs from energy flowing through a system and becoming less available for work." }, { "id": 3110, "name": "ecological restoration", "grade": 2.0, "judgeA": 2, "judgeB": 2, "agreedObligations": [ 0 ], "definition": "Assisting the recovery of an ecosystem that has been damaged, degraded or destroyed through interventions guided by ecological understanding and explicit goals. Recovery need not mean exact reproduction of one historical condition." }, { "id": 3111, "name": "ecosystem service", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "A benefit people obtain from ecosystems, such as food, water regulation, pollination or cultural experience. This human-benefit lens helps describe dependencies while leaving room for ecological and ethical values not captured by service accounting." }, { "id": 3117, "name": "ecosystem", "grade": 2.5, "judgeA": 2, "judgeB": 3, "agreedObligations": [ 1 ], "definition": "Organisms and their physical environment considered as an interacting system, connected by transfers of energy and cycling of materials. Its boundary and scale depend on the question being studied; it is more than a list of species sharing a place." }, { "id": 3120, "name": "net ecosystem production", "grade": 1.0, "judgeA": 1, "judgeB": 1, "agreedObligations": [], "definition": "Gross primary production minus respiration by all organisms within the selected ecosystem and time interval. Other transfers and disturbances must be accounted for before interpreting this difference as a complete carbon-stock change." }, { "id": 3121, "name": "food web", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "A network of feeding relationships that connects organisms through transfers of matter and energy. A participant may draw on several food sources and occupy more than one trophic level, so a single food chain captures only selected paths through the network." }, { "id": 4387, "name": "trophic cascade", "grade": 3.0, "judgeA": 3, "judgeB": 3, "agreedObligations": [ 0, 1 ], "definition": "An indirect ecological effect transmitted across trophic levels, for example when changing predators changes herbivore pressure on producers. Direction and strength depend on the food web and conditions; restoring one species need not guarantee a desired ecosystem outcome." }, { "id": 5801, "name": "wetland", "grade": 0.5, "judgeA": 1, "judgeB": 0, "agreedObligations": [], "definition": "An ecosystem whose sustained or recurrent inundation or saturation shapes soil processes and biota, often producing oxygen-limited soils and water-adapted vegetation." }, { "id": 6188, "name": "snow line", "grade": 0.0, "judgeA": 0, "judgeB": 0, "agreedObligations": [], "definition": "A boundary between snow-covered and snow-free terrain under a specified spatial and temporal definition. Seasonal, end-of-melt-season and persistent snow lines answer different questions." } ] } }, "oracleSubsetsEnumerated": 121036 }, "handoffs": { "overall": { "nAnswers": 768, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, "groups": [ { "stage": 1, "format": "prose-inline", "condition": "complete", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 1, "format": "prose-inline", "condition": "displaced", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 1, "format": "structured-inline", "condition": "complete", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 1, "format": "structured-inline", "condition": "displaced", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 1, "format": "generic-reference", "condition": "complete", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 1, "format": "generic-reference", "condition": "displaced", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 1, "format": "carrier-reference", "condition": "complete", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 1, "format": "carrier-reference", "condition": "displaced", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 2, "format": "prose-inline", "condition": "complete", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 2, "format": "prose-inline", "condition": "displaced", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 2, "format": "structured-inline", "condition": "complete", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 2, "format": "structured-inline", "condition": "displaced", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 2, "format": "generic-reference", "condition": "complete", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 2, "format": "generic-reference", "condition": "displaced", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 2, "format": "carrier-reference", "condition": "complete", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, { "stage": 2, "format": "carrier-reference", "condition": "displaced", "nAnswers": 48, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 } ], "stageTwoUpdates": { "nAnswers": 128, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": null, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, "stageTwoUnmodified": { "nAnswers": 256, "accuracy": 1.0, "knownAccuracy": 1.0, "justifiedUnknown": 1.0, "unsupportedNonUnknownCount": 0, "incorrectUnknownCount": 0 }, "budgetViolations": [], "memoryBytes": { "mean": 708.3515625, "max": 859 }, "pairedExploratory": { "structured-inline": { "deltaVsProse": 0.0, "baseCaseDeltas": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ], "bootstrap95": [ 0.0, 0.0 ], "notEquivalenceTest": true }, "generic-reference": { "deltaVsProse": 0.0, "baseCaseDeltas": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ], "bootstrap95": [ 0.0, 0.0 ], "notEquivalenceTest": true }, "carrier-reference": { "deltaVsProse": 0.0, "baseCaseDeltas": [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ], "bootstrap95": [ 0.0, 0.0 ], "notEquivalenceTest": true } }, "limitations": [ "Eight synthetic base cases; repeated observations are not independent participants.", "These scores cover six closed-world fields, not every free-prose claim.", "Shared definitions are supplementary; case facts are the main answer source.", "Fresh contexts share the configured model family; no authenticated model-build identity.", "No equivalence or general hallucination-rate inference from zero errors." ] }, "referencesAndScale": { "format": "carrier-r03-reference-scale-results/1", "createdUTC": "20261005T121923472822Z", "protocolSHA256": "d5b1a99714914ce6e7d52abec0cc9041cfec6a18f785d26212a08d3fa3da9bcc", "scriptSHA256": "1566fc930cfa2b522664d4cc37d85c4d0dedd7f673f9bb66b7b72f2f91232724", "protocolAmendmentsSHA256": "02f2dba192cd364b133eb38b8cf250129936617d405573be8fc6a633b1ea8d58", "corpus": { "records": 6257, "checkedHead": "398492317d139bd78ca599965f08603296daedd95f4fb088fde05393175bfe88", "conceptsFileSHA256": "3ef29e2f0119bc34ddb549731e525959941fb5e091e8b166775df1c108829f97", "checkedFileSHA256": "cef9836f062ebf9b8bc12f0184716df2a74161be2a09af860247e83cbc4cb89b", "conceptsFileBytes": 2658890, "checkedFileBytes": 9477865, "canonicalEvidenceArrayBytes": 2186495, "verification": "Checked raw-line chain and full evidence projection verified; absent relations normalize to [] as in the frozen R02 concepts file" }, "integrity": { "cases": [ { "scheme": "generic", "case": "valid-cold", "expected": "resolved", "observed": "resolved", "passed": true }, { "scheme": "generic", "case": "valid-warm", "expected": "resolved", "observed": "resolved", "passed": true }, { "scheme": "generic", "case": "wrong-head", "expected": "wrong-head", "observed": "wrong-head", "passed": true }, { "scheme": "generic", "case": "stale-cache-head", "expected": "stale-cache-head", "observed": "stale-cache-head", "passed": true }, { "scheme": "generic", "case": "stale-cache-definition", "expected": "stale-cache-evidence", "observed": "stale-cache-evidence", "passed": true }, { "scheme": "generic", "case": "missing-id", "expected": "missing-id", "observed": "missing-id", "passed": true }, { "scheme": "generic", "case": "mismatched-definition-hash", "expected": "definition-hash-mismatch", "observed": "definition-hash-mismatch", "passed": true }, { "scheme": "generic", "case": "valid-id-unsupported-claim", "expected": "resolved", "observed": "resolved", "passed": true }, { "scheme": "carrier", "case": "valid-cold", "expected": "resolved", "observed": "resolved", "passed": true }, { "scheme": "carrier", "case": "valid-warm", "expected": "resolved", "observed": "resolved", "passed": true }, { "scheme": "carrier", "case": "wrong-head", "expected": "wrong-head", "observed": "wrong-head", "passed": true }, { "scheme": "carrier", "case": "stale-cache-head", "expected": "stale-cache-head", "observed": "stale-cache-head", "passed": true }, { "scheme": "carrier", "case": "stale-cache-definition", "expected": "stale-cache-evidence", "observed": "stale-cache-evidence", "passed": true }, { "scheme": "carrier", "case": "missing-id", "expected": "missing-id", "observed": "missing-id", "passed": true }, { "scheme": "carrier", "case": "mismatched-definition-hash", "expected": "definition-hash-mismatch", "observed": "definition-hash-mismatch", "passed": true }, { "scheme": "carrier", "case": "valid-id-unsupported-claim", "expected": "resolved", "observed": "resolved", "passed": true } ], "passed": 16, "failed": 0, "semanticSupportAssessed": false, "conclusion": "Both namespaces reject the same five integrity failures. Both accept the valid reference carrying an unsupported claim; claim entailment requires a separate semantic assessment. No hallucination rate is measured." }, "costs": { "encoding": "Exact uncompressed UTF-8 canonical JSON application bytes; not tokens or packet capture", "registry": { "generic": { "referenceIndexBytes": 768607, "canonicalEvidenceArrayBytes": 2186495, "fullRegistryProvisionBytes": 2955102 }, "carrier": { "referenceIndexBytes": 768607, "canonicalEvidenceArrayBytes": 2186495, "fullRegistryProvisionBytes": 2955102 } }, "rows": [ { "scheme": "generic", "selectedRecordCount": 3, "selectedIDs": [ 1511, 5628, 178 ], "inlineEvidenceMessageBytes": 986, "referenceMessageBytes": 442, "lookupRequestBytes": 160, "lookupResponseBytes": 986, "warmFromEmptyBreakEvenHandoffs": 3, "warmPlusFullRegistryBreakEvenHandoffs": 5435, "handoffTotals": [ { "handoffs": 1, "inlineWireBytes": 986, "referenceColdWireBytes": 1588, "referenceWarmFromEmptyWireBytes": 1588, "referencePreloadedWireBytes": 442, "referenceWarmPlusFullRegistryWireBytes": 2956690, "resolvedEvidenceBytesAcrossHandoffs": 986, "warmSavingRelativeToInlineBytes": -602, "coldLookupRequests": 1, "warmLookupRequests": 1, "preloadedLookupRequests": 0 }, { "handoffs": 10, "inlineWireBytes": 9860, "referenceColdWireBytes": 15880, "referenceWarmFromEmptyWireBytes": 5566, "referencePreloadedWireBytes": 4420, "referenceWarmPlusFullRegistryWireBytes": 2960668, "resolvedEvidenceBytesAcrossHandoffs": 9860, "warmSavingRelativeToInlineBytes": 4294, "coldLookupRequests": 10, "warmLookupRequests": 1, "preloadedLookupRequests": 0 }, { "handoffs": 100, "inlineWireBytes": 98600, "referenceColdWireBytes": 158800, "referenceWarmFromEmptyWireBytes": 45346, "referencePreloadedWireBytes": 44200, "referenceWarmPlusFullRegistryWireBytes": 3000448, "resolvedEvidenceBytesAcrossHandoffs": 98600, "warmSavingRelativeToInlineBytes": 53254, "coldLookupRequests": 100, "warmLookupRequests": 1, "preloadedLookupRequests": 0 } ] }, { "scheme": "generic", "selectedRecordCount": 32, "selectedIDs": [ 1511, 5628, 178, 5892, 495, 6039, 1847, 2302, 2609, 4264, 5236, 4457, 3116, 4021, 4337, 6027, 3938, 2282, 3000, 101, 1399, 1685, 4034, 5014, 5811, 5851, 4220, 4443, 4531, 6058, 2907, 4591 ], "inlineEvidenceMessageBytes": 10889, "referenceMessageBytes": 3719, "lookupRequestBytes": 711, "lookupResponseBytes": 10889, "warmFromEmptyBreakEvenHandoffs": 2, "warmPlusFullRegistryBreakEvenHandoffs": 414, "handoffTotals": [ { "handoffs": 1, "inlineWireBytes": 10889, "referenceColdWireBytes": 15319, "referenceWarmFromEmptyWireBytes": 15319, "referencePreloadedWireBytes": 3719, "referenceWarmPlusFullRegistryWireBytes": 2970421, "resolvedEvidenceBytesAcrossHandoffs": 10889, "warmSavingRelativeToInlineBytes": -4430, "coldLookupRequests": 1, "warmLookupRequests": 1, "preloadedLookupRequests": 0 }, { "handoffs": 10, "inlineWireBytes": 108890, "referenceColdWireBytes": 153190, "referenceWarmFromEmptyWireBytes": 48790, "referencePreloadedWireBytes": 37190, "referenceWarmPlusFullRegistryWireBytes": 3003892, "resolvedEvidenceBytesAcrossHandoffs": 108890, "warmSavingRelativeToInlineBytes": 60100, "coldLookupRequests": 10, "warmLookupRequests": 1, "preloadedLookupRequests": 0 }, { "handoffs": 100, "inlineWireBytes": 1088900, "referenceColdWireBytes": 1531900, "referenceWarmFromEmptyWireBytes": 383500, "referencePreloadedWireBytes": 371900, "referenceWarmPlusFullRegistryWireBytes": 3338602, "resolvedEvidenceBytesAcrossHandoffs": 1088900, "warmSavingRelativeToInlineBytes": 705400, "coldLookupRequests": 100, "warmLookupRequests": 1, "preloadedLookupRequests": 0 } ] }, { "scheme": "generic", "selectedRecordCount": 100, "selectedIDs": [ 1511, 5628, 178, 5892, 495, 6039, 1847, 2302, 2609, 4264, 5236, 4457, 3116, 4021, 4337, 6027, 3938, 2282, 3000, 101, 1399, 1685, 4034, 5014, 5811, 5851, 4220, 4443, 4531, 6058, 2907, 4591, 4745, 4117, 5758, 805, 5171, 2546, 330, 5274, 3409, 4731, 2755, 3053, 2300, 1930, 1944, 2554, 1818, 4535, 5435, 2330, 2962, 5601, 1245, 1514, 4761, 263, 4783, 5303, 5993, 628, 141, 5206, 2345, 3857, 3546, 5055, 4864, 4024, 4306, 4285, 2564, 2207, 3195, 295, 308, 733, 5598, 1882, 950, 4796, 3735, 2061, 5826, 5742, 2872, 1604, 3901, 4835, 2007, 5958, 4419, 3022, 2104, 833, 1619, 6024, 2144, 3148 ], "inlineEvidenceMessageBytes": 34222, "referenceMessageBytes": 11403, "lookupRequestBytes": 2003, "lookupResponseBytes": 34222, "warmFromEmptyBreakEvenHandoffs": 2, "warmPlusFullRegistryBreakEvenHandoffs": 132, "handoffTotals": [ { "handoffs": 1, "inlineWireBytes": 34222, "referenceColdWireBytes": 47628, "referenceWarmFromEmptyWireBytes": 47628, "referencePreloadedWireBytes": 11403, "referenceWarmPlusFullRegistryWireBytes": 3002730, "resolvedEvidenceBytesAcrossHandoffs": 34222, "warmSavingRelativeToInlineBytes": -13406, "coldLookupRequests": 1, "warmLookupRequests": 1, "preloadedLookupRequests": 0 }, { "handoffs": 10, "inlineWireBytes": 342220, "referenceColdWireBytes": 476280, "referenceWarmFromEmptyWireBytes": 150255, "referencePreloadedWireBytes": 114030, "referenceWarmPlusFullRegistryWireBytes": 3105357, "resolvedEvidenceBytesAcrossHandoffs": 342220, "warmSavingRelativeToInlineBytes": 191965, "coldLookupRequests": 10, "warmLookupRequests": 1, "preloadedLookupRequests": 0 }, { "handoffs": 100, "inlineWireBytes": 3422200, "referenceColdWireBytes": 4762800, "referenceWarmFromEmptyWireBytes": 1176525, "referencePreloadedWireBytes": 1140300, "referenceWarmPlusFullRegistryWireBytes": 4131627, "resolvedEvidenceBytesAcrossHandoffs": 3422200, "warmSavingRelativeToInlineBytes": 2245675, "coldLookupRequests": 100, "warmLookupRequests": 1, "preloadedLookupRequests": 0 } ] }, { "scheme": "carrier", "selectedRecordCount": 3, "selectedIDs": [ 1511, 5628, 178 ], "inlineEvidenceMessageBytes": 986, "referenceMessageBytes": 442, "lookupRequestBytes": 160, "lookupResponseBytes": 986, "warmFromEmptyBreakEvenHandoffs": 3, "warmPlusFullRegistryBreakEvenHandoffs": 5435, "handoffTotals": [ { "handoffs": 1, "inlineWireBytes": 986, "referenceColdWireBytes": 1588, "referenceWarmFromEmptyWireBytes": 1588, "referencePreloadedWireBytes": 442, "referenceWarmPlusFullRegistryWireBytes": 2956690, "resolvedEvidenceBytesAcrossHandoffs": 986, "warmSavingRelativeToInlineBytes": -602, "coldLookupRequests": 1, "warmLookupRequests": 1, "preloadedLookupRequests": 0 }, { "handoffs": 10, "inlineWireBytes": 9860, "referenceColdWireBytes": 15880, "referenceWarmFromEmptyWireBytes": 5566, "referencePreloadedWireBytes": 4420, "referenceWarmPlusFullRegistryWireBytes": 2960668, "resolvedEvidenceBytesAcrossHandoffs": 9860, "warmSavingRelativeToInlineBytes": 4294, "coldLookupRequests": 10, "warmLookupRequests": 1, "preloadedLookupRequests": 0 }, { "handoffs": 100, "inlineWireBytes": 98600, "referenceColdWireBytes": 158800, "referenceWarmFromEmptyWireBytes": 45346, "referencePreloadedWireBytes": 44200, "referenceWarmPlusFullRegistryWireBytes": 3000448, "resolvedEvidenceBytesAcrossHandoffs": 98600, "warmSavingRelativeToInlineBytes": 53254, "coldLookupRequests": 100, "warmLookupRequests": 1, "preloadedLookupRequests": 0 } ] }, { "scheme": "carrier", "selectedRecordCount": 32, "selectedIDs": [ 1511, 5628, 178, 5892, 495, 6039, 1847, 2302, 2609, 4264, 5236, 4457, 3116, 4021, 4337, 6027, 3938, 2282, 3000, 101, 1399, 1685, 4034, 5014, 5811, 5851, 4220, 4443, 4531, 6058, 2907, 4591 ], "inlineEvidenceMessageBytes": 10889, "referenceMessageBytes": 3719, "lookupRequestBytes": 711, "lookupResponseBytes": 10889, "warmFromEmptyBreakEvenHandoffs": 2, "warmPlusFullRegistryBreakEvenHandoffs": 414, "handoffTotals": [ { "handoffs": 1, "inlineWireBytes": 10889, "referenceColdWireBytes": 15319, "referenceWarmFromEmptyWireBytes": 15319, "referencePreloadedWireBytes": 3719, "referenceWarmPlusFullRegistryWireBytes": 2970421, "resolvedEvidenceBytesAcrossHandoffs": 10889, "warmSavingRelativeToInlineBytes": -4430, "coldLookupRequests": 1, "warmLookupRequests": 1, "preloadedLookupRequests": 0 }, { "handoffs": 10, "inlineWireBytes": 108890, "referenceColdWireBytes": 153190, "referenceWarmFromEmptyWireBytes": 48790, "referencePreloadedWireBytes": 37190, "referenceWarmPlusFullRegistryWireBytes": 3003892, "resolvedEvidenceBytesAcrossHandoffs": 108890, "warmSavingRelativeToInlineBytes": 60100, "coldLookupRequests": 10, "warmLookupRequests": 1, "preloadedLookupRequests": 0 }, { "handoffs": 100, "inlineWireBytes": 1088900, "referenceColdWireBytes": 1531900, "referenceWarmFromEmptyWireBytes": 383500, "referencePreloadedWireBytes": 371900, "referenceWarmPlusFullRegistryWireBytes": 3338602, "resolvedEvidenceBytesAcrossHandoffs": 1088900, "warmSavingRelativeToInlineBytes": 705400, "coldLookupRequests": 100, "warmLookupRequests": 1, "preloadedLookupRequests": 0 } ] }, { "scheme": "carrier", "selectedRecordCount": 100, "selectedIDs": [ 1511, 5628, 178, 5892, 495, 6039, 1847, 2302, 2609, 4264, 5236, 4457, 3116, 4021, 4337, 6027, 3938, 2282, 3000, 101, 1399, 1685, 4034, 5014, 5811, 5851, 4220, 4443, 4531, 6058, 2907, 4591, 4745, 4117, 5758, 805, 5171, 2546, 330, 5274, 3409, 4731, 2755, 3053, 2300, 1930, 1944, 2554, 1818, 4535, 5435, 2330, 2962, 5601, 1245, 1514, 4761, 263, 4783, 5303, 5993, 628, 141, 5206, 2345, 3857, 3546, 5055, 4864, 4024, 4306, 4285, 2564, 2207, 3195, 295, 308, 733, 5598, 1882, 950, 4796, 3735, 2061, 5826, 5742, 2872, 1604, 3901, 4835, 2007, 5958, 4419, 3022, 2104, 833, 1619, 6024, 2144, 3148 ], "inlineEvidenceMessageBytes": 34222, "referenceMessageBytes": 11403, "lookupRequestBytes": 2003, "lookupResponseBytes": 34222, "warmFromEmptyBreakEvenHandoffs": 2, "warmPlusFullRegistryBreakEvenHandoffs": 132, "handoffTotals": [ { "handoffs": 1, "inlineWireBytes": 34222, "referenceColdWireBytes": 47628, "referenceWarmFromEmptyWireBytes": 47628, "referencePreloadedWireBytes": 11403, "referenceWarmPlusFullRegistryWireBytes": 3002730, "resolvedEvidenceBytesAcrossHandoffs": 34222, "warmSavingRelativeToInlineBytes": -13406, "coldLookupRequests": 1, "warmLookupRequests": 1, "preloadedLookupRequests": 0 }, { "handoffs": 10, "inlineWireBytes": 342220, "referenceColdWireBytes": 476280, "referenceWarmFromEmptyWireBytes": 150255, "referencePreloadedWireBytes": 114030, "referenceWarmPlusFullRegistryWireBytes": 3105357, "resolvedEvidenceBytesAcrossHandoffs": 342220, "warmSavingRelativeToInlineBytes": 191965, "coldLookupRequests": 10, "warmLookupRequests": 1, "preloadedLookupRequests": 0 }, { "handoffs": 100, "inlineWireBytes": 3422200, "referenceColdWireBytes": 4762800, "referenceWarmFromEmptyWireBytes": 1176525, "referencePreloadedWireBytes": 1140300, "referenceWarmPlusFullRegistryWireBytes": 4131627, "resolvedEvidenceBytesAcrossHandoffs": 3422200, "warmSavingRelativeToInlineBytes": 2245675, "coldLookupRequests": 100, "warmLookupRequests": 1, "preloadedLookupRequests": 0 } ] } ], "conclusion": "Equal-length generic and Carrier references have identical payload cost. Caching may save repeated wire bytes while every receiver still needs the materialized evidence. Full-registry distribution is accounted separately." }, "environment": { "python": "3.12.14 (main, Aug 25 2026, 14:01:42) [MSC v.1944 64 bit (AMD64)]", "platform": "Windows-11-10.0.26200-SP0", "cpuCount": 20, "processor": "Intel64 Family 6 Model 154 Stepping 3, GenuineIntel", "pythonHashSeed": "randomized/default", "timer": "namespace(implementation='QueryPerformanceCounter()', monotonic=True, adjustable=False, resolution=1e-07)", "gcEnabled": true }, "selfTest": { "passed": true, "canonicalEncoding": true, "strictBreakEvenBoundary": true, "syntheticStorageFormula": true }, "scale": { "sharedBaseDefinitionStringBytes": 1897756, "rows": [ { "syntheticUniqueKeys": 6257, "baseDefinitionRecords": 6257, "distinctBaseDefinitionTexts": 6256, "additionalDefinitionTexts": 0, "repetitions": [ { "repeat": 1, "buildMilliseconds": 1.3261, "lookupBatchMilliseconds": 1.6266, "lookupNanosecondsPerQuery": 81.33, "queries": 20000, "hits": 10000, "misses": 10000, "checksum": 2335668, "dictionaryShallowBytes": 207616, "ownedKeyStringBytes": 356649, "dictionaryPlusOwnedKeysBytes": 564265 }, { "repeat": 2, "buildMilliseconds": 1.0695, "lookupBatchMilliseconds": 1.6229, "lookupNanosecondsPerQuery": 81.145, "queries": 20000, "hits": 10000, "misses": 10000, "checksum": 2332664, "dictionaryShallowBytes": 207616, "ownedKeyStringBytes": 356649, "dictionaryPlusOwnedKeysBytes": 564265 }, { "repeat": 3, "buildMilliseconds": 1.0876, "lookupBatchMilliseconds": 1.6024, "lookupNanosecondsPerQuery": 80.12, "queries": 20000, "hits": 10000, "misses": 10000, "checksum": 2330853, "dictionaryShallowBytes": 207616, "ownedKeyStringBytes": 356649, "dictionaryPlusOwnedKeysBytes": 564265 } ], "medianBuildMilliseconds": 1.0876, "medianLookupNanosecondsPerQuery": 81.145, "minLookupNanosecondsPerQuery": 80.12, "maxLookupNanosecondsPerQuery": 81.33, "syntheticJSONLIndexBytes": 255427 }, { "syntheticUniqueKeys": 100000, "baseDefinitionRecords": 6257, "distinctBaseDefinitionTexts": 6256, "additionalDefinitionTexts": 0, "repetitions": [ { "repeat": 1, "buildMilliseconds": 23.2376, "lookupBatchMilliseconds": 4.441, "lookupNanosecondsPerQuery": 222.05, "queries": 20000, "hits": 10000, "misses": 10000, "checksum": 2330461, "dictionaryShallowBytes": 3844864, "ownedKeyStringBytes": 5700000, "dictionaryPlusOwnedKeysBytes": 9544864 }, { "repeat": 2, "buildMilliseconds": 24.5567, "lookupBatchMilliseconds": 4.1495, "lookupNanosecondsPerQuery": 207.475, "queries": 20000, "hits": 10000, "misses": 10000, "checksum": 2330221, "dictionaryShallowBytes": 3844864, "ownedKeyStringBytes": 5700000, "dictionaryPlusOwnedKeysBytes": 9544864 }, { "repeat": 3, "buildMilliseconds": 22.3712, "lookupBatchMilliseconds": 3.8113, "lookupNanosecondsPerQuery": 190.565, "queries": 20000, "hits": 10000, "misses": 10000, "checksum": 2334216, "dictionaryShallowBytes": 3844864, "ownedKeyStringBytes": 5700000, "dictionaryPlusOwnedKeysBytes": 9544864 } ], "medianBuildMilliseconds": 23.2376, "medianLookupNanosecondsPerQuery": 207.475, "minLookupNanosecondsPerQuery": 190.565, "maxLookupNanosecondsPerQuery": 222.05, "syntheticJSONLIndexBytes": 4082240 }, { "syntheticUniqueKeys": 1000000, "baseDefinitionRecords": 6257, "distinctBaseDefinitionTexts": 6256, "additionalDefinitionTexts": 0, "repetitions": [ { "repeat": 1, "buildMilliseconds": 359.0864, "lookupBatchMilliseconds": 6.4816, "lookupNanosecondsPerQuery": 324.08, "queries": 20000, "hits": 10000, "misses": 10000, "checksum": 2333893, "dictionaryShallowBytes": 30758320, "ownedKeyStringBytes": 57000000, "dictionaryPlusOwnedKeysBytes": 87758320 }, { "repeat": 2, "buildMilliseconds": 300.5677, "lookupBatchMilliseconds": 6.0166, "lookupNanosecondsPerQuery": 300.83, "queries": 20000, "hits": 10000, "misses": 10000, "checksum": 2338290, "dictionaryShallowBytes": 30758320, "ownedKeyStringBytes": 57000000, "dictionaryPlusOwnedKeysBytes": 87758320 }, { "repeat": 3, "buildMilliseconds": 308.8186, "lookupBatchMilliseconds": 6.7769, "lookupNanosecondsPerQuery": 338.845, "queries": 20000, "hits": 10000, "misses": 10000, "checksum": 2340195, "dictionaryShallowBytes": 30758320, "ownedKeyStringBytes": 57000000, "dictionaryPlusOwnedKeysBytes": 87758320 } ], "medianBuildMilliseconds": 308.8186, "medianLookupNanosecondsPerQuery": 324.08, "minLookupNanosecondsPerQuery": 300.83, "maxLookupNanosecondsPerQuery": 338.845, "syntheticJSONLIndexBytes": 40822400 } ], "timingScope": "Local in-process dictionary build and mixed hit/miss get loop; excludes hashing, source loading, network, semantic retrieval, full resolver validation and LLM operations", "memoryScope": "Python shallow dictionary plus unique key strings, with shared base definition strings reported separately. Excludes process/interpreter overhead, query arrays, other corpus objects, allocator fragmentation and peak RSS.", "limitation": "6257 meanings reused under up to one million synthetic keys; no additional knowledge and no answer-accuracy extrapolation. OS caches, CPU load, garbage collection, allocator state and Python hash randomization can affect results." }, "competitionModel": { "status": "Analytical assumption illustration, not measured retrieval accuracy", "assumptions": "Essential score fixed at .999; independent Uniform(0,1) distractor scores; top5 iff at most four distractors exceed .999", "rows": [ { "N": 6257, "distractors": 6256, "expectedOutrankingDistractors": 6.256, "top1Log10Probability": -2.7183056582546765, "top5Log10Probability": -0.5984073475084165, "top5Probability": 0.2521114977559475, "underflowNotation": "Use log10Probability when ordinary probability is null" }, { "N": 100000, "distractors": 99999, "expectedOutrankingDistractors": 99.999, "top1Log10Probability": -43.45074288999511, "top5Log10Probability": -36.81173225732695, "top5Probability": 1.5426512047616316e-37, "underflowNotation": "Use log10Probability when ordinary probability is null" }, { "N": 1000000, "distractors": 999999, "expectedOutrankingDistractors": 999.999, "top1Log10Probability": -434.51133950591725, "top5Log10Probability": -423.88807986324025, "top5Probability": null, "underflowNotation": "Use log10Probability when ordinary probability is null" } ] } } }