{
  "format": "carrier-task-retrieval-r02/1",
  "title": "Carrier Retrieval Lab R02 — Questions before concepts",
  "status": "Locally prospective protocol, before question collection, pooling and scoring; not external preregistration",
  "corpus": "Unchanged 6,257 checked records from R01, head 398492317d139bd78ca599965f08603296daedd95f4fb088fde05393175bfe88. R01 questions and relevance targets are not used.",
  "questions": {
    "count": 48,
    "domains": [
      "mathematics",
      "physics",
      "statistics",
      "philosophy",
      "biology",
      "economics"
    ],
    "perDomain": "Four actual public human questions adapted with attribution, plus four prospectively authored AI scenario tasks. Source wording and adaptations kept separate. Real questions are a purposive sample, not a population sample of Carrier users.",
    "independence": "Two isolated author processes without Carrier concepts, links, targets, ranks or R01 questions. Public answers are not sought; unsolicited answer exposure is disclosed. No claim of human-authored adaptations, independent human relevance judgments or external replication.",
    "rubricBeforeRetrieval": "Each task specifies 3–5 useful answer obligations and explicit conditions, before candidate exposure.",
    "freeze": "First complete task files hashed, provenance validated. Content corrections retained if needed; never based on retrieval outcome.",
    "split": "Within each domain and origin, use seeded shuffle 1282026: one of four development, three final test. 12 development and 36 final. Split hidden from candidate relevance judges and readers. Original human question and any derived variants kept in one split. No editing after freeze."
  },
  "retrieval": {
    "baselines": [
      "BM25 on repeated name plus definition, fixed k1=1.2 b=.75",
      "Pinned local MiniLM cosine, same R01 document representations, no R01 query vectors",
      "Positive-score reciprocal-rank fusion BM25 plus MiniLM, k=60"
    ],
    "baselineSelection": "Highest mean development pooled nDCG@5; tie order BM25, hybrid, dense.",
    "contextGraph": "Original untyped undirected contextual graph, top20 seeds, 20 walk iterations, teleport=.7, blend grid [.1,.3], fixed before outcomes.",
    "typedOverlay": "Independent definition-based directed typed research overlay built without access to questions. Literal evidence spans, scope qualifications, rule identity, extraction limitations and abstentions retained. Not promoted Carrier records, not universal logical entailment.",
    "typedGrid": "Directed paths of one or two hops with predicate and query-role weights; top20 positive seeds, path-length decay .5. Blend [.1,.3]. No untyped fallback; no typed path means exact baseline identity. Query-intent rules, type weights and implementation freeze before pools or scores. Select highest development nDCG@5, ties smaller blend then shorter paths.",
    "controls": "Untyped graph; typed-edge direction reversed; type labels permuted with endpoints fixed; empty typed graph. Same selected typed parameters. Controls are secondary and cannot replace the main method after final results.",
    "ranking": "Positive scores only, checked-ID ascending tie resolution. All eligible records searchable. No relevance labels used by retrieval."
  },
  "pooling": {
    "rule": "Union top five from every fixed candidate setting for every task, plus lexical ranks6 and10 and one seeded random corpus item. Include all control top5. No relevance prefilter. Pool identity uses opaque task-local candidate codes; random order independently for two assessors. Hide method, ranks, split, graph degree and checked ID.",
    "judging": "Two isolated AI assessors grade every pooled item 0 irrelevant, 1 related/background, 2 useful for an explicit obligation, 3 directly supports a central answer distinction. Record obligation coverage and short rationale. No accepted-answer reading. Related-looking grade0/1 items retained as hard-negative candidates; no claim they are universally irrelevant.",
    "combination": "Arithmetic mean of the two ordinal grades used as a declared soft relevance score; no post-outcome adjudication. Report exact agreement, quadratic weighted kappa, individual-judge sensitivity and disputed items. Full pool remains available for human rejudgment.",
    "metricLimit": "nDCG@5 ideal is over this incomplete method-pooled set. All compared top5 judged. Not exhaustive recall, universal answer truth, or unbiased relevance to all future systems."
  },
  "primary": {
    "retrieval": "Mean final-query pooled nDCG@5: development-selected typed method minus development-selected text/dense baseline. Gain 2^meanGrade−1. Queries with zero positive pool ideal contribute0 and remain counted.",
    "answer": "Randomized two-reader task on all36 final questions. Each reader sees each question only once, with either baseline or selected typed top5 context; assignments complementary and balanced three per method within each domain. Context documents retain their retrieval order; method identity and relevance judgments hidden. Both readers use only supplied definitions, mark insufficiency and cite opaque context labels.",
    "answerScoring": "Two independent graders see question, prewritten obligations, supplied context and answer, with reader/method identities hidden and shuffled ordering. Score each obligation0/1/2, role/condition fidelity0/1/2, and evidence discipline0/1/2. Normalize total to[0,1]. Record unsupported claims, false citations and qualified abstention separately. Graders assess answer utility and grounding, not quotation matching alone. Mean grader score is endpoint; all raw responses preserved before grading.",
    "multiplicity": "Two co-primary contrasts, Holm adjustment at familywise .05; report raw and adjusted p-values. No selective choice between retrieval and answer endpoint."
  },
  "statistics": {
    "bootstrap": "10000 paired query bootstrap replicates seed128300, percentile95% intervals; exploratory stratified-by-domain sensitivity",
    "retrievalTest": "10000 two-sided paired sign-flips, plus-one correction",
    "answerRandomization": "Random allocation seed128400 balanced3/3 within six domain blocks; randomization p enumerates or samples valid within-domain assignments against fixed observed reader scores under the sharp null. Paired bootstrap describes observed question-level method differences. Two reader processes, not36 independent human readers.",
    "secondary": "Human-origin and AI-origin strata; domain strata; obligation coverage@5; grade>=2 precision@5; untyped/reversed/permuted controls; judge disagreement; selected ranks and path support; lexical/dense overlap; answer citations/unsupported claims; per-reader assignment balance. No unjudged item counted as confirmed negative."
  },
  "integrity": "Freeze task texts, overlay, candidate algorithms and grading rules before pooling. Freeze graded pools before development selection; final qrels may be generated beforehand but final metrics not computed/inspected until parameter lock. Parent orchestrator cannot be fully blinded; selection scripts use only development rows. Final scoring refuses overwrite. Preserve failures and repairs. No Carrier mutations or live deployment.",
  "limitations": "Same underlying model family may supply writers, judges and readers. Separate contexts reduce direct leakage but do not make agents independent humans or remove pretraining contamination. Public source questions may appear in training. All conclusions conditional on purposive tasks, model-generated labels, one frozen corpus and this implementation. Typed overlay extraction is independently auditable but not human-validated.",
  "sources": [
    {
      "title": "Assessing relevance — Introduction to Information Retrieval",
      "url": "https://nlp.stanford.edu/IR-book/html/htmledition/assessing-relevance-1.html"
    },
    {
      "title": "Evaluation of ranked retrieval results",
      "url": "https://nlp.stanford.edu/IR-book/html/htmledition/evaluation-of-ranked-retrieval-results-1.html"
    },
    {
      "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
      "url": "https://arxiv.org/abs/2306.05685"
    },
    {
      "title": "Lost in the Middle: How Language Models Use Long Contexts",
      "url": "https://arxiv.org/abs/2307.03172"
    }
  ]
}