{ "createdUTC": "2026-10-04T22:22:44.981681+00:00", "queries": 36, "primary": { "n": 36, "difference": -0.0034016948777831745, "ci95": [ -0.010205084633349524, 0.0 ], "twoSidedSignFlipP": 1.0, "wins": 0, "losses": 1, "ties": 35, "bootstrapReplicates": 10000, "seed": 128300 }, "stratifiedBootstrap": { "n": 36, "difference": -0.0034016948777831745, "ci95": [ -0.010205084633349524, 0.0 ], "twoSidedSignFlipP": 1.0, "wins": 0, "losses": 1, "ties": 35, "bootstrapReplicates": 10000, "seed": 128300 }, "lock": { "lockedUTC": "2026-10-04T22:22:33.366341+00:00", "baseline": "dense", "typed": "typed:dense:1:0.1", "untyped": "untyped:dense:0.1", "developmentQueries": 12, "selection": "Development mean pooled nDCG5; fixed tie order", "files": { "benchmark/tasks.json": "8c9a285410bcf9d43b2028438ebb9f5e04b34803aa0fac9e4ddea34811bac39f", "data/typed-overlay.json": "b5e704a9eaa589c44fca76600d6fca3d760338fa5e3d7623732ffdd86015c26f", "analysis/qrels.json": "078c72c16df0dd97cf1ea409d38a22a78940236adc7e50b5c54666083eace325", "analysis/candidate-rankings.json": "7a4bc03a4b9080e8e547f9ae599e5081c128851ed8b3e9f41f8d529c461f6180", "tools/typed_retrieval.py": "dd48ea635a583e33b62e9b762928d1d9083e652e4a5f3dd4e258bb28457bcf42", "tools/evaluate_retrieval.py": "cafe184d1468ad36297a0a2a5dc65b33ac0b350aad3116f0ddbe4cb45adae64f", "tools/metrics_r02.py": "1200c91a0a4ce129e1b95dd9c4d6fbf5dc7478cc69e7265b91c3ea99f72f2b7e" } }, "metrics": { "bm25": { "ndcg5": 0.6099465412644016, "precision5Grade2": 0.5222222222222221, "agreedObligationCoverage5": 0.6805555555555556 }, "dense": { "ndcg5": 0.6740797549180285, "precision5Grade2": 0.6000000000000001, "agreedObligationCoverage5": 0.7361111111111112 }, "hybrid": { "ndcg5": 0.7407360466471675, "precision5Grade2": 0.6277777777777779, "agreedObligationCoverage5": 0.7847222222222222 }, "untyped:bm25:0.1": { "ndcg5": 0.6139068427950142, "precision5Grade2": 0.5222222222222221, "agreedObligationCoverage5": 0.6805555555555556 }, "untyped:bm25:0.3": { "ndcg5": 0.6130006998483758, "precision5Grade2": 0.5388888888888889, "agreedObligationCoverage5": 0.6875 }, "typed:bm25:1:0.1": { "ndcg5": 0.6132828344573854, "precision5Grade2": 0.5333333333333332, "agreedObligationCoverage5": 0.6875 }, "typed:bm25:1:0.3": { "ndcg5": 0.6080892353770205, "precision5Grade2": 0.5333333333333332, "agreedObligationCoverage5": 0.6875 }, "typed:bm25:2:0.1": { "ndcg5": 0.6132828344573854, "precision5Grade2": 0.5333333333333332, "agreedObligationCoverage5": 0.6875 }, "typed:bm25:2:0.3": { "ndcg5": 0.6080892353770205, "precision5Grade2": 0.5333333333333332, "agreedObligationCoverage5": 0.6875 }, "reverse:bm25:1:0.1": { "ndcg5": 0.6160564322306392, "precision5Grade2": 0.5277777777777778, "agreedObligationCoverage5": 0.6805555555555556 }, "reverse:bm25:1:0.3": { "ndcg5": 0.6141748148727915, "precision5Grade2": 0.5388888888888889, "agreedObligationCoverage5": 0.6875 }, "reverse:bm25:2:0.1": { "ndcg5": 0.6160564322306392, "precision5Grade2": 0.5277777777777778, "agreedObligationCoverage5": 0.6805555555555556 }, "reverse:bm25:2:0.3": { "ndcg5": 0.6141748148727915, "precision5Grade2": 0.5388888888888889, "agreedObligationCoverage5": 0.6875 }, "permuted:bm25:1:0.1": { "ndcg5": 0.6132828344573854, "precision5Grade2": 0.5333333333333332, "agreedObligationCoverage5": 0.6875 }, "permuted:bm25:1:0.3": { "ndcg5": 0.6080892353770205, "precision5Grade2": 0.5333333333333332, "agreedObligationCoverage5": 0.6875 }, "permuted:bm25:2:0.1": { "ndcg5": 0.6132828344573854, "precision5Grade2": 0.5333333333333332, "agreedObligationCoverage5": 0.6875 }, "permuted:bm25:2:0.3": { "ndcg5": 0.6080892353770205, "precision5Grade2": 0.5333333333333332, "agreedObligationCoverage5": 0.6875 }, "empty:bm25:1:0.1": { "ndcg5": 0.6099465412644016, "precision5Grade2": 0.5222222222222221, "agreedObligationCoverage5": 0.6805555555555556 }, "empty:bm25:1:0.3": { "ndcg5": 0.6099465412644016, "precision5Grade2": 0.5222222222222221, "agreedObligationCoverage5": 0.6805555555555556 }, "empty:bm25:2:0.1": { "ndcg5": 0.6099465412644016, "precision5Grade2": 0.5222222222222221, "agreedObligationCoverage5": 0.6805555555555556 }, "empty:bm25:2:0.3": { "ndcg5": 0.6099465412644016, "precision5Grade2": 0.5222222222222221, "agreedObligationCoverage5": 0.6805555555555556 }, "untyped:dense:0.1": { "ndcg5": 0.6851918344157654, "precision5Grade2": 0.6222222222222222, "agreedObligationCoverage5": 0.7361111111111112 }, "untyped:dense:0.3": { "ndcg5": 0.6946234008067382, "precision5Grade2": 0.6166666666666667, "agreedObligationCoverage5": 0.7430555555555556 }, "typed:dense:1:0.1": { "ndcg5": 0.6706780600402453, "precision5Grade2": 0.6000000000000001, "agreedObligationCoverage5": 0.7291666666666666 }, "typed:dense:1:0.3": { "ndcg5": 0.6780810812107513, "precision5Grade2": 0.6111111111111112, "agreedObligationCoverage5": 0.7361111111111112 }, "typed:dense:2:0.1": { "ndcg5": 0.6706780600402453, "precision5Grade2": 0.6000000000000001, "agreedObligationCoverage5": 0.7291666666666666 }, "typed:dense:2:0.3": { "ndcg5": 0.6780810812107513, "precision5Grade2": 0.6111111111111112, "agreedObligationCoverage5": 0.7361111111111112 }, "reverse:dense:1:0.1": { "ndcg5": 0.666329006690069, "precision5Grade2": 0.5944444444444446, "agreedObligationCoverage5": 0.7291666666666666 }, "reverse:dense:1:0.3": { "ndcg5": 0.6593998936267254, "precision5Grade2": 0.5944444444444446, "agreedObligationCoverage5": 0.7361111111111112 }, "reverse:dense:2:0.1": { "ndcg5": 0.666329006690069, "precision5Grade2": 0.5944444444444446, "agreedObligationCoverage5": 0.7291666666666666 }, "reverse:dense:2:0.3": { "ndcg5": 0.6593998936267254, "precision5Grade2": 0.5944444444444446, "agreedObligationCoverage5": 0.7361111111111112 }, "permuted:dense:1:0.1": { "ndcg5": 0.6706780600402453, "precision5Grade2": 0.6000000000000001, "agreedObligationCoverage5": 0.7291666666666666 }, "permuted:dense:1:0.3": { "ndcg5": 0.6780810812107513, "precision5Grade2": 0.6111111111111112, "agreedObligationCoverage5": 0.7361111111111112 }, "permuted:dense:2:0.1": { "ndcg5": 0.6706780600402453, "precision5Grade2": 0.6000000000000001, "agreedObligationCoverage5": 0.7291666666666666 }, "permuted:dense:2:0.3": { "ndcg5": 0.6780810812107513, "precision5Grade2": 0.6111111111111112, "agreedObligationCoverage5": 0.7361111111111112 }, "empty:dense:1:0.1": { "ndcg5": 0.6740797549180285, "precision5Grade2": 0.6000000000000001, "agreedObligationCoverage5": 0.7361111111111112 }, "empty:dense:1:0.3": { "ndcg5": 0.6740797549180285, "precision5Grade2": 0.6000000000000001, "agreedObligationCoverage5": 0.7361111111111112 }, "empty:dense:2:0.1": { "ndcg5": 0.6740797549180285, "precision5Grade2": 0.6000000000000001, "agreedObligationCoverage5": 0.7361111111111112 }, "empty:dense:2:0.3": { "ndcg5": 0.6740797549180285, "precision5Grade2": 0.6000000000000001, "agreedObligationCoverage5": 0.7361111111111112 }, "untyped:hybrid:0.1": { "ndcg5": 0.7522106041107604, "precision5Grade2": 0.6444444444444446, "agreedObligationCoverage5": 0.7847222222222222 }, "untyped:hybrid:0.3": { "ndcg5": 0.7505627199881713, "precision5Grade2": 0.6555555555555557, "agreedObligationCoverage5": 0.7916666666666666 }, "typed:hybrid:1:0.1": { "ndcg5": 0.7354785226340721, "precision5Grade2": 0.6277777777777779, "agreedObligationCoverage5": 0.7847222222222222 }, "typed:hybrid:1:0.3": { "ndcg5": 0.7252365799466091, "precision5Grade2": 0.6277777777777778, "agreedObligationCoverage5": 0.7708333333333334 }, "typed:hybrid:2:0.1": { "ndcg5": 0.7354785226340721, "precision5Grade2": 0.6277777777777779, "agreedObligationCoverage5": 0.7847222222222222 }, "typed:hybrid:2:0.3": { "ndcg5": 0.7252365799466091, "precision5Grade2": 0.6277777777777778, "agreedObligationCoverage5": 0.7708333333333334 }, "reverse:hybrid:1:0.1": { "ndcg5": 0.7398891441861098, "precision5Grade2": 0.6277777777777779, "agreedObligationCoverage5": 0.7847222222222222 }, "reverse:hybrid:1:0.3": { "ndcg5": 0.7304602889684277, "precision5Grade2": 0.6277777777777778, "agreedObligationCoverage5": 0.7847222222222222 }, "reverse:hybrid:2:0.1": { "ndcg5": 0.7398891441861098, "precision5Grade2": 0.6277777777777779, "agreedObligationCoverage5": 0.7847222222222222 }, "reverse:hybrid:2:0.3": { "ndcg5": 0.7304602889684277, "precision5Grade2": 0.6277777777777778, "agreedObligationCoverage5": 0.7847222222222222 }, "permuted:hybrid:1:0.1": { "ndcg5": 0.7354785226340721, "precision5Grade2": 0.6277777777777779, "agreedObligationCoverage5": 0.7847222222222222 }, "permuted:hybrid:1:0.3": { "ndcg5": 0.7252365799466091, "precision5Grade2": 0.6277777777777778, "agreedObligationCoverage5": 0.7708333333333334 }, "permuted:hybrid:2:0.1": { "ndcg5": 0.7354785226340721, "precision5Grade2": 0.6277777777777779, "agreedObligationCoverage5": 0.7847222222222222 }, "permuted:hybrid:2:0.3": { "ndcg5": 0.7252365799466091, "precision5Grade2": 0.6277777777777778, "agreedObligationCoverage5": 0.7708333333333334 }, "empty:hybrid:1:0.1": { "ndcg5": 0.7407360466471675, "precision5Grade2": 0.6277777777777779, "agreedObligationCoverage5": 0.7847222222222222 }, "empty:hybrid:1:0.3": { "ndcg5": 0.7407360466471675, "precision5Grade2": 0.6277777777777779, "agreedObligationCoverage5": 0.7847222222222222 }, "empty:hybrid:2:0.1": { "ndcg5": 0.7407360466471675, "precision5Grade2": 0.6277777777777779, "agreedObligationCoverage5": 0.7847222222222222 }, "empty:hybrid:2:0.3": { "ndcg5": 0.7407360466471675, "precision5Grade2": 0.6277777777777779, "agreedObligationCoverage5": 0.7847222222222222 } }, "selected": [ "dense", "typed:dense:1:0.1", "untyped:dense:0.1", "reverse:dense:1:0.1", "permuted:dense:1:0.1", "empty:dense:1:0.1" ], "rankings": { "dense": [ { "task": "A02", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.6082561650962087, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3381, 4512, 4348, 4513, 3557 ] }, { "task": "A03", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.7155302697944608, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 4466, 4465, 3344, 4468, 4467 ] }, { "task": "A04", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.974679678148328, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3333, 4638, 4495, 4640, 4492 ] }, { "task": "A05", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.5304759541593448, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3580, 3579, 4399, 3416, 4768 ] }, { "task": "A06", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.49301520849026176, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3568, 3565, 4376, 3569, 3635 ] }, { "task": "A07", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.6533463439927868, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3548, 3686, 3680, 6065, 2340 ] }, { "task": "A09", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.9177513973666739, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4692, 4694, 3424, 3494, 4693 ] }, { "task": "A10", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.9058527658710546, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3442, 2466, 4565, 4667, 4650 ] }, { "task": "A11", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.7645295418043103, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3498, 4608, 3496, 4636, 4625 ] }, { "task": "A14", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.7422601353304841, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2410, 2723, 2724, 2844, 4738 ] }, { "task": "A15", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.9232684181649756, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3461, 2547, 3390, 3455, 3458 ] }, { "task": "A16", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.9030661971488, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2219, 2274, 3495, 3496, 4629 ] }, { "task": "A17", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.6795674418464838, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3553, 2421, 3556, 6035, 3647 ] }, { "task": "A19", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.9142126528861984, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5947, 1234, 5950, 2850, 5958 ] }, { "task": "A20", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.5464913089438267, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 2102, 3189, 5965, 6191, 3579 ] }, { "task": "A21", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.4710468685170683, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3449, 3472, 3475, 3478, 4699 ] }, { "task": "A23", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.8114968423174953, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4197, 3456, 3461, 1882, 3492 ] }, { "task": "A24", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.6285608281560209, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3432, 4565, 4564, 4563, 3442 ] }, { "task": "B01", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.6978872952515099, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3836, 2003, 3832, 3938, 2012 ] }, { "task": "B03", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.27344128348213886, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 3817, 2055, 3869, 4236, 1578 ] }, { "task": "B04", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.6995577437011554, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3817, 2055, 3869, 5040, 3863 ] }, { "task": "B05", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.6534046191206433, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3106, 4341, 3125, 3103, 1697 ] }, { "task": "B06", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.5114237404054409, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 771, 4413, 5845, 5824, 1951 ] }, { "task": "B08", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.7982296316051806, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3106, 1825, 5766, 1281, 5771 ] }, { "task": "B10", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.680725678280414, "precision5Grade2": 1.0, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5065, 1608, 5062, 5067, 1482 ] }, { "task": "B11", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.6303994075567672, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3874, 5101, 2204, 4318, 5264 ] }, { "task": "B12", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.70807361818795, "precision5Grade2": 0.0, "agreedObligationCoverage5": 0.0, "positiveIdeal": true, "top5": [ 3222, 4868, 3876, 3079, 2885 ] }, { "task": "B13", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.6940209635364101, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 1836, 2247, 1675, 595, 874 ] }, { "task": "B14", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.7014941890986883, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 125, 2128, 1080, 1715, 5028 ] }, { "task": "B15", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.8066031581233893, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4078, 983, 5042, 4079, 4081 ] }, { "task": "B17", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.6072827569747501, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 3108, 3109, 5779, 3103, 1824 ] }, { "task": "B18", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.47697017530379393, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3111, 3090, 3120, 75, 3077 ] }, { "task": "B20", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.4698115003932712, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3103, 3125, 2918, 1824, 75 ] }, { "task": "B21", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.5761076079923059, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 2162, 4377, 1599, 917, 2148 ] }, { "task": "B22", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.4485515486456346, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4379, 3876, 1596, 2896, 5092 ] }, { "task": "B23", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.6494782413548044, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 944, 3967, 2473, 1034, 5049 ] } ], "typed:dense:1:0.1": [ { "task": "A02", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.6082561650962087, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3381, 4512, 4348, 4513, 3557 ] }, { "task": "A03", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.7155302697944608, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 4466, 4465, 3344, 4468, 4467 ] }, { "task": "A04", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.974679678148328, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3333, 4638, 4495, 4640, 4492 ] }, { "task": "A05", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.5304759541593448, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3580, 3579, 4399, 3416, 4768 ] }, { "task": "A06", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.49301520849026176, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3568, 3565, 4376, 3569, 3635 ] }, { "task": "A07", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.6533463439927868, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3548, 3686, 3680, 6065, 2340 ] }, { "task": "A09", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.9177513973666739, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4692, 4694, 3424, 3494, 4693 ] }, { "task": "A10", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.9058527658710546, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3442, 2466, 4565, 4667, 4650 ] }, { "task": "A11", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.7645295418043103, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3498, 4608, 3496, 4636, 4625 ] }, { "task": "A14", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.7422601353304841, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2410, 2723, 2724, 2844, 4738 ] }, { "task": "A15", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.9232684181649756, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3461, 2547, 3390, 3455, 3458 ] }, { "task": "A16", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.9030661971488, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2219, 2274, 3495, 3496, 4629 ] }, { "task": "A17", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.6795674418464838, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3553, 2421, 3556, 6035, 3647 ] }, { "task": "A19", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.9142126528861984, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5947, 1234, 5950, 2850, 5958 ] }, { "task": "A20", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.5464913089438267, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 2102, 3189, 5965, 6191, 3579 ] }, { "task": "A21", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.348585852916874, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3472, 3449, 3475, 3478, 4699 ] }, { "task": "A23", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.8114968423174953, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4197, 3456, 3461, 1882, 3492 ] }, { "task": "A24", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.6285608281560209, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3432, 4565, 4564, 4563, 3442 ] }, { "task": "B01", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.6978872952515099, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3836, 2003, 3832, 3938, 2012 ] }, { "task": "B03", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.27344128348213886, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 3817, 2055, 3869, 4236, 1578 ] }, { "task": "B04", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.6995577437011554, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3817, 2055, 3869, 5040, 3863 ] }, { "task": "B05", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.6534046191206433, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3106, 4341, 3125, 3103, 1697 ] }, { "task": "B06", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.5114237404054409, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 771, 4413, 5845, 5824, 1951 ] }, { "task": "B08", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.7982296316051806, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3106, 1825, 5766, 1281, 5771 ] }, { "task": "B10", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.680725678280414, "precision5Grade2": 1.0, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5065, 1608, 5062, 5067, 1482 ] }, { "task": "B11", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.6303994075567672, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3874, 5101, 2204, 4318, 5264 ] }, { "task": "B12", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.70807361818795, "precision5Grade2": 0.0, "agreedObligationCoverage5": 0.0, "positiveIdeal": true, "top5": [ 3222, 4868, 3876, 3079, 2885 ] }, { "task": "B13", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.6940209635364101, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 1836, 2247, 1675, 595, 874 ] }, { "task": "B14", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.7014941890986883, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 125, 2128, 1080, 1715, 5028 ] }, { "task": "B15", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.8066031581233893, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4078, 983, 5042, 4079, 4081 ] }, { "task": "B17", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.6072827569747501, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 3108, 3109, 5779, 3103, 1824 ] }, { "task": "B18", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.47697017530379393, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3111, 3090, 3120, 75, 3121 ] }, { "task": "B20", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.4698115003932712, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3103, 3125, 2918, 1824, 75 ] }, { "task": "B21", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.5761076079923059, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 2162, 4377, 1599, 917, 2148 ] }, { "task": "B22", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.4485515486456346, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4379, 3876, 1596, 2896, 5092 ] }, { "task": "B23", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.6494782413548044, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 944, 3967, 2473, 1034, 5049 ] } ], "untyped:dense:0.1": [ { "task": "A02", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.6082561650962087, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3381, 4512, 4348, 4513, 3557 ] }, { "task": "A03", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.7155302697944608, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 4466, 4465, 3344, 4468, 4467 ] }, { "task": "A04", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.974679678148328, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3333, 4638, 4495, 4640, 4492 ] }, { "task": "A05", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.5304759541593448, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3580, 3579, 4399, 3416, 4768 ] }, { "task": "A06", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.5445862125193615, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3568, 3565, 4376, 3570, 3569 ] }, { "task": "A07", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.6533463439927868, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3686, 3548, 3680, 6065, 2340 ] }, { "task": "A09", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.9177513973666739, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4692, 4694, 3424, 3494, 4693 ] }, { "task": "A10", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.9083719543569371, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3442, 2466, 4565, 4650, 4667 ] }, { "task": "A11", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.8129417798226305, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3498, 4608, 3496, 4636, 3501 ] }, { "task": "A14", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.7422601353304841, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2410, 2723, 2724, 2844, 4738 ] }, { "task": "A15", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.9145930178368985, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3461, 2547, 3455, 3390, 3458 ] }, { "task": "A16", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.9030661971488, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2219, 3495, 2274, 3496, 4629 ] }, { "task": "A17", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.6795674418464838, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3553, 2421, 3556, 6035, 3647 ] }, { "task": "A19", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.904266089415881, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5947, 1234, 2850, 5950, 5958 ] }, { "task": "A20", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.5464913089438267, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2102, 3189, 5965, 6191, 3577 ] }, { "task": "A21", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.4710468685170683, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3449, 3472, 3475, 3478, 4699 ] }, { "task": "A23", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.8114968423174953, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4197, 3456, 3461, 1882, 3492 ] }, { "task": "A24", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.6909002666933918, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3432, 4565, 4564, 4563, 3431 ] }, { "task": "B01", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.7897078692027677, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3836, 2003, 3832, 3820, 3938 ] }, { "task": "B03", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.433525845032428, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 3817, 2055, 3869, 4236, 4235 ] }, { "task": "B04", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.6995577437011554, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3817, 2055, 3869, 5040, 3863 ] }, { "task": "B05", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.7283789491276957, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3106, 4341, 3103, 3125, 3124 ] }, { "task": "B06", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.5114237404054409, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 771, 4413, 5845, 5824, 1951 ] }, { "task": "B08", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.7982296316051806, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3106, 1825, 5766, 1281, 5771 ] }, { "task": "B10", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.680725678280414, "precision5Grade2": 1.0, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5065, 1608, 5062, 5067, 1482 ] }, { "task": "B11", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.6327399668579062, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3874, 5101, 2204, 5264, 4318 ] }, { "task": "B12", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.70807361818795, "precision5Grade2": 0.0, "agreedObligationCoverage5": 0.0, "positiveIdeal": true, "top5": [ 3222, 4868, 3876, 3079, 2885 ] }, { "task": "B13", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.6940209635364101, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 1836, 2247, 1675, 595, 874 ] }, { "task": "B14", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.7014941890986883, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 125, 2128, 1080, 1715, 5028 ] }, { "task": "B15", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.7667922817509988, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4078, 5042, 983, 4079, 4081 ] }, { "task": "B17", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.6072827569747501, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 3108, 3109, 5779, 3103, 1824 ] }, { "task": "B18", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.4310337502588347, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3111, 3090, 3120, 75, 3117 ] }, { "task": "B20", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.4698115003932712, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3103, 3125, 2918, 1824, 75 ] }, { "task": "B21", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.5761076079923059, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 2162, 4377, 1599, 917, 2148 ] }, { "task": "B22", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.4588937818994953, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4379, 3876, 1596, 5092, 2896 ] }, { "task": "B23", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.6494782413548044, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 944, 3967, 2473, 1034, 5049 ] } ], "reverse:dense:1:0.1": [ { "task": "A02", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.6082561650962087, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3381, 4512, 4348, 4513, 3557 ] }, { "task": "A03", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.7155302697944608, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 4466, 4465, 3344, 4468, 4467 ] }, { "task": "A04", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.9472254378497933, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3333, 4638, 4495, 4640, 4502 ] }, { "task": "A05", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.5304759541593448, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3580, 3579, 4399, 3416, 4768 ] }, { "task": "A06", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.49301520849026176, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3568, 3565, 4376, 3569, 3635 ] }, { "task": "A07", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.6533463439927868, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3548, 3686, 3680, 6065, 2340 ] }, { "task": "A09", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.9177513973666739, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4692, 4694, 3424, 3494, 4693 ] }, { "task": "A10", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.9058527658710546, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3442, 2466, 4565, 4667, 4650 ] }, { "task": "A11", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.7645295418043103, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3498, 4608, 3496, 4636, 4625 ] }, { "task": "A14", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.7422601353304841, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2410, 2723, 2724, 2844, 4738 ] }, { "task": "A15", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.9232684181649756, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3461, 2547, 3390, 3455, 3458 ] }, { "task": "A16", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.9030661971488, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2219, 2274, 3495, 3496, 4629 ] }, { "task": "A17", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.6795674418464838, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3553, 2421, 3556, 6035, 3647 ] }, { "task": "A19", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.9142126528861984, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5947, 1234, 5950, 2850, 5958 ] }, { "task": "A20", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.5464913089438267, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 2102, 3189, 5965, 6191, 3579 ] }, { "task": "A21", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.348585852916874, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3475, 3449, 3472, 3478, 4699 ] }, { "task": "A23", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.8114968423174953, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4197, 3456, 3461, 1882, 3492 ] }, { "task": "A24", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.6285608281560209, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3432, 4565, 4564, 4563, 3442 ] }, { "task": "B01", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.6978872952515099, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3836, 2003, 3832, 3938, 2012 ] }, { "task": "B03", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.27344128348213886, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 3817, 2055, 3869, 4236, 1578 ] }, { "task": "B04", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.6995577437011554, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3817, 2055, 3869, 5040, 3863 ] }, { "task": "B05", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.6534046191206433, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3106, 4341, 3125, 3103, 1697 ] }, { "task": "B06", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.5114237404054409, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 771, 4413, 5845, 5824, 1951 ] }, { "task": "B08", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.7982296316051806, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3106, 1825, 5766, 1281, 5771 ] }, { "task": "B10", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.680725678280414, "precision5Grade2": 1.0, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5065, 1608, 5062, 5067, 1482 ] }, { "task": "B11", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.6303994075567672, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3874, 5101, 2204, 4318, 5264 ] }, { "task": "B12", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.70807361818795, "precision5Grade2": 0.0, "agreedObligationCoverage5": 0.0, "positiveIdeal": true, "top5": [ 3222, 4868, 3876, 3079, 2885 ] }, { "task": "B13", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.6940209635364101, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 1836, 2247, 1675, 595, 874 ] }, { "task": "B14", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.7014941890986883, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 125, 2128, 1080, 1715, 5028 ] }, { "task": "B15", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.8066031581233893, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4078, 983, 5042, 4079, 4081 ] }, { "task": "B17", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.6072827569747501, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 3108, 3109, 5779, 3103, 1824 ] }, { "task": "B18", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.34785849499597793, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3111, 3090, 3120, 75, 5801 ] }, { "task": "B20", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.4698115003932712, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3103, 3125, 2918, 1824, 75 ] }, { "task": "B21", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.5761076079923059, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 2162, 4377, 1599, 917, 2148 ] }, { "task": "B22", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.4485515486456346, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4379, 3876, 1596, 2896, 5092 ] }, { "task": "B23", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.6494782413548044, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 944, 3967, 2473, 1034, 5049 ] } ], "permuted:dense:1:0.1": [ { "task": "A02", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.6082561650962087, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3381, 4512, 4348, 4513, 3557 ] }, { "task": "A03", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.7155302697944608, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 4466, 4465, 3344, 4468, 4467 ] }, { "task": "A04", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.974679678148328, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3333, 4638, 4495, 4640, 4492 ] }, { "task": "A05", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.5304759541593448, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3580, 3579, 4399, 3416, 4768 ] }, { "task": "A06", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.49301520849026176, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3568, 3565, 4376, 3569, 3635 ] }, { "task": "A07", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.6533463439927868, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3548, 3686, 3680, 6065, 2340 ] }, { "task": "A09", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.9177513973666739, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4692, 4694, 3424, 3494, 4693 ] }, { "task": "A10", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.9058527658710546, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3442, 2466, 4565, 4667, 4650 ] }, { "task": "A11", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.7645295418043103, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3498, 4608, 3496, 4636, 4625 ] }, { "task": "A14", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.7422601353304841, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2410, 2723, 2724, 2844, 4738 ] }, { "task": "A15", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.9232684181649756, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3461, 2547, 3390, 3455, 3458 ] }, { "task": "A16", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.9030661971488, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2219, 2274, 3495, 3496, 4629 ] }, { "task": "A17", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.6795674418464838, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3553, 2421, 3556, 6035, 3647 ] }, { "task": "A19", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.9142126528861984, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5947, 1234, 5950, 2850, 5958 ] }, { "task": "A20", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.5464913089438267, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 2102, 3189, 5965, 6191, 3579 ] }, { "task": "A21", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.348585852916874, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3472, 3449, 3475, 3478, 4699 ] }, { "task": "A23", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.8114968423174953, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4197, 3456, 3461, 1882, 3492 ] }, { "task": "A24", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.6285608281560209, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3432, 4565, 4564, 4563, 3442 ] }, { "task": "B01", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.6978872952515099, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3836, 2003, 3832, 3938, 2012 ] }, { "task": "B03", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.27344128348213886, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 3817, 2055, 3869, 4236, 1578 ] }, { "task": "B04", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.6995577437011554, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3817, 2055, 3869, 5040, 3863 ] }, { "task": "B05", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.6534046191206433, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3106, 4341, 3125, 3103, 1697 ] }, { "task": "B06", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.5114237404054409, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 771, 4413, 5845, 5824, 1951 ] }, { "task": "B08", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.7982296316051806, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3106, 1825, 5766, 1281, 5771 ] }, { "task": "B10", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.680725678280414, "precision5Grade2": 1.0, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5065, 1608, 5062, 5067, 1482 ] }, { "task": "B11", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.6303994075567672, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3874, 5101, 2204, 4318, 5264 ] }, { "task": "B12", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.70807361818795, "precision5Grade2": 0.0, "agreedObligationCoverage5": 0.0, "positiveIdeal": true, "top5": [ 3222, 4868, 3876, 3079, 2885 ] }, { "task": "B13", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.6940209635364101, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 1836, 2247, 1675, 595, 874 ] }, { "task": "B14", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.7014941890986883, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 125, 2128, 1080, 1715, 5028 ] }, { "task": "B15", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.8066031581233893, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4078, 983, 5042, 4079, 4081 ] }, { "task": "B17", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.6072827569747501, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 3108, 3109, 5779, 3103, 1824 ] }, { "task": "B18", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.47697017530379393, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3111, 3090, 3120, 75, 3121 ] }, { "task": "B20", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.4698115003932712, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3103, 3125, 2918, 1824, 75 ] }, { "task": "B21", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.5761076079923059, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 2162, 4377, 1599, 917, 2148 ] }, { "task": "B22", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.4485515486456346, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4379, 3876, 1596, 2896, 5092 ] }, { "task": "B23", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.6494782413548044, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 944, 3967, 2473, 1034, 5049 ] } ], "empty:dense:1:0.1": [ { "task": "A02", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.6082561650962087, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3381, 4512, 4348, 4513, 3557 ] }, { "task": "A03", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.7155302697944608, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 4466, 4465, 3344, 4468, 4467 ] }, { "task": "A04", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.974679678148328, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3333, 4638, 4495, 4640, 4492 ] }, { "task": "A05", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.5304759541593448, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3580, 3579, 4399, 3416, 4768 ] }, { "task": "A06", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.49301520849026176, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3568, 3565, 4376, 3569, 3635 ] }, { "task": "A07", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.6533463439927868, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3548, 3686, 3680, 6065, 2340 ] }, { "task": "A09", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.9177513973666739, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4692, 4694, 3424, 3494, 4693 ] }, { "task": "A10", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.9058527658710546, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3442, 2466, 4565, 4667, 4650 ] }, { "task": "A11", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.7645295418043103, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3498, 4608, 3496, 4636, 4625 ] }, { "task": "A14", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.7422601353304841, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2410, 2723, 2724, 2844, 4738 ] }, { "task": "A15", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.9232684181649756, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3461, 2547, 3390, 3455, 3458 ] }, { "task": "A16", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.9030661971488, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2219, 2274, 3495, 3496, 4629 ] }, { "task": "A17", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.6795674418464838, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3553, 2421, 3556, 6035, 3647 ] }, { "task": "A19", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.9142126528861984, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5947, 1234, 5950, 2850, 5958 ] }, { "task": "A20", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.5464913089438267, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 2102, 3189, 5965, 6191, 3579 ] }, { "task": "A21", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.4710468685170683, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3449, 3472, 3475, 3478, 4699 ] }, { "task": "A23", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.8114968423174953, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4197, 3456, 3461, 1882, 3492 ] }, { "task": "A24", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.6285608281560209, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3432, 4565, 4564, 4563, 3442 ] }, { "task": "B01", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.6978872952515099, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3836, 2003, 3832, 3938, 2012 ] }, { "task": "B03", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.27344128348213886, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 3817, 2055, 3869, 4236, 1578 ] }, { "task": "B04", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.6995577437011554, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3817, 2055, 3869, 5040, 3863 ] }, { "task": "B05", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.6534046191206433, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3106, 4341, 3125, 3103, 1697 ] }, { "task": "B06", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.5114237404054409, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 771, 4413, 5845, 5824, 1951 ] }, { "task": "B08", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.7982296316051806, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3106, 1825, 5766, 1281, 5771 ] }, { "task": "B10", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.680725678280414, "precision5Grade2": 1.0, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5065, 1608, 5062, 5067, 1482 ] }, { "task": "B11", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.6303994075567672, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3874, 5101, 2204, 4318, 5264 ] }, { "task": "B12", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.70807361818795, "precision5Grade2": 0.0, "agreedObligationCoverage5": 0.0, "positiveIdeal": true, "top5": [ 3222, 4868, 3876, 3079, 2885 ] }, { "task": "B13", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.6940209635364101, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 1836, 2247, 1675, 595, 874 ] }, { "task": "B14", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.7014941890986883, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 125, 2128, 1080, 1715, 5028 ] }, { "task": "B15", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.8066031581233893, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4078, 983, 5042, 4079, 4081 ] }, { "task": "B17", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.6072827569747501, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 3108, 3109, 5779, 3103, 1824 ] }, { "task": "B18", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.47697017530379393, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3111, 3090, 3120, 75, 3077 ] }, { "task": "B20", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.4698115003932712, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3103, 3125, 2918, 1824, 75 ] }, { "task": "B21", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.5761076079923059, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 2162, 4377, 1599, 917, 2148 ] }, { "task": "B22", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.4485515486456346, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4379, 3876, 1596, 2896, 5092 ] }, { "task": "B23", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.6494782413548044, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 944, 3967, 2473, 1034, 5049 ] } ], "bm25": [ { "task": "A02", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.47283179762400007, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3381, 3373, 3346, 3343, 4483 ] }, { "task": "A03", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.851176061066942, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3344, 3340, 4974, 3309, 4447 ] }, { "task": "A04", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.9037114964908282, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3333, 4638, 4642, 4653, 4497 ] }, { "task": "A05", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.42558207359578865, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 3579, 3058, 3572, 3612, 6047 ] }, { "task": "A06", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.7738992005821805, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 2836, 3588, 4373, 3570, 2815 ] }, { "task": "A07", "domain": "physics", "origin": "public-human-question", "ndcg5": 1.0, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3680, 3681, 3561, 3686, 3548 ] }, { "task": "A09", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.7653593583044233, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4692, 3437, 4623, 4694, 4618 ] }, { "task": "A10", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.5653246586168539, "precision5Grade2": 0.2, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3442, 4554, 5752, 4579, 4650 ] }, { "task": "A11", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.8403308123696069, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3498, 3499, 3501, 4538, 4599 ] }, { "task": "A14", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.8192269140253222, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 2723, 2724, 4738, 2721, 1886 ] }, { "task": "A15", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.3396364572481077, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 2219, 4627, 2509, 1910, 2351 ] }, { "task": "A16", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.9165323715184533, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3493, 3333, 3495, 3456, 3338 ] }, { "task": "A17", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.4174304585908707, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 6000, 6164, 3553, 3477, 2956 ] }, { "task": "A19", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.03729579494676704, "precision5Grade2": 0.0, "agreedObligationCoverage5": 0.0, "positiveIdeal": true, "top5": [ 5751, 1917, 2145, 2259, 4470 ] }, { "task": "A20", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.5575514942601042, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3564, 4394, 3563, 1262, 5998 ] }, { "task": "A21", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.8620465361311771, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3428, 3449, 3429, 4684, 4288 ] }, { "task": "A23", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.7279930278412972, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4578, 1882, 4608, 3456, 4641 ] }, { "task": "A24", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.5201714686245358, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 5253, 4557, 4766, 3458, 4563 ] }, { "task": "B01", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.6812734240573711, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3836, 2003, 3847, 5008, 1941 ] }, { "task": "B03", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.4536753625867996, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3817, 2010, 4231, 4236, 1586 ] }, { "task": "B04", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.23980173319115133, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4991, 1334, 3817, 4926, 3622 ] }, { "task": "B05", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.8148424570560182, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3106, 2346, 5785, 5767, 3098 ] }, { "task": "B06", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.2900103773677023, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 5108, 4456, 4067, 2105, 3467 ] }, { "task": "B08", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.754220089922232, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3106, 1693, 1825, 5498, 3104 ] }, { "task": "B10", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.8682612561552213, "precision5Grade2": 1.0, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5073, 1482, 5065, 5091, 5059 ] }, { "task": "B11", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.9158858640715238, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3874, 5099, 5101, 2048, 2138 ] }, { "task": "B12", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.344481216982244, "precision5Grade2": 0.0, "agreedObligationCoverage5": 0.0, "positiveIdeal": true, "top5": [ 1794, 2752, 3079, 2545, 5094 ] }, { "task": "B13", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.5473942369675174, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 4191, 2247, 1675, 760, 2167 ] }, { "task": "B14", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.5215869002134366, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5028, 2128, 1080, 1757, 36 ] }, { "task": "B15", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.22969509404339133, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 4851, 1291, 3810, 2062, 2208 ] }, { "task": "B17", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.8347580537913257, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 5773, 5507, 5493, 3082, 3108 ] }, { "task": "B18", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.2787242614170329, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 1970, 6188, 1558, 1124, 3121 ] }, { "task": "B20", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.6451307345945306, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 4387, 6201, 84, 1639, 3099 ] }, { "task": "B21", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.39314629139898954, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 5335, 4384, 1199, 815, 6154 ] }, { "task": "B22", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.7756757319942509, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 1609, 5106, 1675, 5085, 5082 ] }, { "task": "B23", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.5734124178704589, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3938, 2473, 939, 299, 1781 ] } ], "hybrid": [ { "task": "A02", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.46135159815921506, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3381, 3346, 4513, 4747, 4512 ] }, { "task": "A03", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.9331784109252725, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3344, 4468, 3340, 4447, 4466 ] }, { "task": "A04", "domain": "mathematics", "origin": "public-human-question", "ndcg5": 0.9648401609087868, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3333, 4638, 4497, 4495, 4653 ] }, { "task": "A05", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.7414101646263457, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3579, 4381, 3510, 3612, 3572 ] }, { "task": "A06", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.6244407360017832, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3565, 2836, 3570, 3569, 3566 ] }, { "task": "A07", "domain": "physics", "origin": "public-human-question", "ndcg5": 0.8227692327421103, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3680, 3548, 3686, 6065, 3681 ] }, { "task": "A09", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.9358192808548432, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4692, 4694, 3424, 3437, 1478 ] }, { "task": "A10", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.7108674790642718, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3442, 4650, 3432, 4563, 4592 ] }, { "task": "A11", "domain": "statistics", "origin": "public-human-question", "ndcg5": 0.8500762529774437, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3498, 3501, 4599, 4636, 3499 ] }, { "task": "A14", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.8139454775732392, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 2723, 2724, 4738, 4737, 2721 ] }, { "task": "A15", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.41101039347799256, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 2351, 2219, 3597, 3460, 3390 ] }, { "task": "A16", "domain": "mathematics", "origin": "prospective-ai-task", "ndcg5": 0.8920852691457679, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3495, 2219, 2274, 4629, 3496 ] }, { "task": "A17", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.822868165813806, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3553, 2421, 3556, 6009, 3511 ] }, { "task": "A19", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.8128996288378502, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5950, 5947, 6035, 2021, 5959 ] }, { "task": "A20", "domain": "physics", "origin": "prospective-ai-task", "ndcg5": 0.5714548078419106, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3189, 3564, 4394, 5965, 1262 ] }, { "task": "A21", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.6393751436186159, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3449, 3475, 4684, 3460, 3450 ] }, { "task": "A23", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.9250256699929476, "precision5Grade2": 1.0, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 1882, 3456, 4578, 3459, 3492 ] }, { "task": "A24", "domain": "statistics", "origin": "prospective-ai-task", "ndcg5": 0.9078588280574262, "precision5Grade2": 1.0, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 4563, 4564, 3432, 3431, 4557 ] }, { "task": "B01", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.7897771417710575, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3836, 2003, 3820, 3847, 3938 ] }, { "task": "B03", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.40562966487278235, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.5, "positiveIdeal": true, "top5": [ 3817, 4236, 1578, 2105, 4230 ] }, { "task": "B04", "domain": "philosophy", "origin": "public-human-question", "ndcg5": 0.9159588702828384, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3817, 4235, 1691, 3853, 2078 ] }, { "task": "B05", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.6534046191206433, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3106, 4341, 3098, 5767, 3110 ] }, { "task": "B06", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.19928518005851156, "precision5Grade2": 0.2, "agreedObligationCoverage5": 0.25, "positiveIdeal": true, "top5": [ 3109, 5844, 5846, 771, 1276 ] }, { "task": "B08", "domain": "biology", "origin": "public-human-question", "ndcg5": 0.9243150395538607, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3106, 1825, 3103, 5766, 3104 ] }, { "task": "B10", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.8375769675791765, "precision5Grade2": 1.0, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 5065, 5073, 1482, 5063, 5062 ] }, { "task": "B11", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.8669668790402674, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 3874, 5101, 2204, 5099, 5100 ] }, { "task": "B12", "domain": "economics", "origin": "public-human-question", "ndcg5": 0.8859704713637417, "precision5Grade2": 0.0, "agreedObligationCoverage5": 0.0, "positiveIdeal": true, "top5": [ 3079, 2885, 5094, 2752, 5093 ] }, { "task": "B13", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.8835914376504294, "precision5Grade2": 0.6, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2247, 1675, 979, 595, 511 ] }, { "task": "B14", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.7144015741005567, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2128, 5028, 1080, 5026, 3845 ] }, { "task": "B15", "domain": "philosophy", "origin": "prospective-ai-task", "ndcg5": 0.3799150045256242, "precision5Grade2": 0.4, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 1291, 4080, 2151, 4078, 961 ] }, { "task": "B17", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.937660561462629, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 3108, 5773, 3082, 5768, 2913 ] }, { "task": "B18", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.6109208076723043, "precision5Grade2": 0.8, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 1558, 4387, 3110, 3111, 3121 ] }, { "task": "B20", "domain": "biology", "origin": "prospective-ai-task", "ndcg5": 0.6957231265819638, "precision5Grade2": 0.4, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 75, 4387, 3099, 3103, 84 ] }, { "task": "B21", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.7563043611939962, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 2148, 4384, 2162, 1599, 3219 ] }, { "task": "B22", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.5897927396252187, "precision5Grade2": 0.6, "agreedObligationCoverage5": 0.75, "positiveIdeal": true, "top5": [ 1675, 2911, 4379, 3922, 2886 ] }, { "task": "B23", "domain": "economics", "origin": "prospective-ai-task", "ndcg5": 0.7780265322228017, "precision5Grade2": 0.8, "agreedObligationCoverage5": 1.0, "positiveIdeal": true, "top5": [ 2473, 3938, 2423, 2121, 5236 ] } ] }, "subgroups": { "domain": { "biology": { "n": 6, "difference": 0.0, "ci95": [ 0.0, 0.0 ], "twoSidedSignFlipP": 1.0, "wins": 0, "losses": 0, "ties": 6, "bootstrapReplicates": 10000, "seed": 128300 }, "economics": { "n": 6, "difference": 0.0, "ci95": [ 0.0, 0.0 ], "twoSidedSignFlipP": 1.0, "wins": 0, "losses": 0, "ties": 6, "bootstrapReplicates": 10000, "seed": 128300 }, "mathematics": { "n": 6, "difference": 0.0, "ci95": [ 0.0, 0.0 ], "twoSidedSignFlipP": 1.0, "wins": 0, "losses": 0, "ties": 6, "bootstrapReplicates": 10000, "seed": 128300 }, "philosophy": { "n": 6, "difference": 0.0, "ci95": [ 0.0, 0.0 ], "twoSidedSignFlipP": 1.0, "wins": 0, "losses": 0, "ties": 6, "bootstrapReplicates": 10000, "seed": 128300 }, "physics": { "n": 6, "difference": 0.0, "ci95": [ 0.0, 0.0 ], "twoSidedSignFlipP": 1.0, "wins": 0, "losses": 0, "ties": 6, "bootstrapReplicates": 10000, "seed": 128300 }, "statistics": { "n": 6, "difference": -0.020410169266699047, "ci95": [ -0.061230507800097145, 0.0 ], "twoSidedSignFlipP": 1.0, "wins": 0, "losses": 1, "ties": 5, "bootstrapReplicates": 10000, "seed": 128300 } }, "origin": { "prospective-ai-task": { "n": 18, "difference": -0.006803389755566349, "ci95": [ -0.020410169266699047, 0.0 ], "twoSidedSignFlipP": 1.0, "wins": 0, "losses": 1, "ties": 17, "bootstrapReplicates": 10000, "seed": 128300 }, "public-human-question": { "n": 18, "difference": 0.0, "ci95": [ 0.0, 0.0 ], "twoSidedSignFlipP": 1.0, "wins": 0, "losses": 0, "ties": 18, "bootstrapReplicates": 10000, "seed": 128300 } } }, "judgeSensitivity": { "judgeA": { "baseline": 0.6612210467620097, "typed": 0.6576796012875598, "paired": { "n": 36, "difference": -0.0035414454744497523, "ci95": [ -0.010624336423349257, 0.0 ], "twoSidedSignFlipP": 1.0, "wins": 0, "losses": 1, "ties": 35, "bootstrapReplicates": 10000, "seed": 128300 } }, "judgeB": { "baseline": 0.6925070608665713, "typed": 0.6892851698269205, "paired": { "n": 36, "difference": -0.003221891039650898, "ci95": [ -0.009665673118952695, 0.0 ], "twoSidedSignFlipP": 1.0, "wins": 0, "losses": 1, "ties": 35, "bootstrapReplicates": 10000, "seed": 128300 } } }, "zeroIdealQueries": [], "inference": "Raw primary p awaits Holm adjustment jointly with answer endpoint. Query-level model-judged inference, not human validation." }