{
  "format": "carrier-retrieval-study-protocol/1",
  "study": "Carrier Retrieval Lab · R01",
  "status": "Local prospective protocol, frozen before retrieval scoring; not externally registered or externally timestamped",
  "snapshotHead": "398492317d139bd78ca599965f08603296daedd95f4fb088fde05393175bfe88",
  "population": "6257 checked concept records in the approved Wave III snapshot. Working records excluded. No dictionary or relation edits during study.",
  "sampling": {"seed":1272026,"targets":120,"strata":"Six equal-width sequential-ID bands; twenty sampled eligible definitions per band. This balances historical position, not subject domains.","eligibility":"At least twelve whitespace-separated definition words; no exclusion by graph degree or retrieval result.","developmentFamilies":40,"heldOutFamilies":80,"queryKinds":["paraphrase","application","boundary"],"queries":360},
  "authorship": "Two AI author processes see only assigned target names and definitions; no graph, rankings, other vocabulary or split. Gold is the assigned intended concept, not an exhaustive relevance judgment. Authors share this project and model family; no independent human or blinded external semantic validation is claimed.",
  "benchmarkFreeze": "Validate IDs, query completeness, title leakage and duplicate queries before scoring. Retain first versions and a changes log if repaired. Hash all queries and split before development scoring. Test outcomes must not be inspected until parameters are locked.",
  "text": "Names repeated twice followed by definition. ASCII alphanumeric lowercase tokens; documented fixed stop list; no stemming, source notes, provenance, IDs, relation labels or full source articles indexed.",
  "baselines": {"bm25":{"k1":1.2,"b":0.75,"idf":"ln(1+(N-df+0.5)/(df+0.5))","queryWeight":"unique terms"},"tfidf":{"tf":"1+ln(count)","idf":"ln((N+1)/(df+1))+1","normalization":"cosine"},"rrf":{"inputs":["bm25","tfidf"],"constant":60,"scope":"positive-scoring documents only"},"lexicalSelection":"Highest development family-mean MRR@10, ties prefer bm25 then tfidf then rrf"},
  "graph": {"definition":"Deduplicated undirected contextual pairs derived from checked relations; no self loops. Stored direction is publication structure, not necessarily a semantic predicate. No inference of equivalence, entailment or permission.","seeds":20,"iterations":20,"teleportGrid":[0.5,0.8],"blendGrid":[0.1,0.25,0.5],"calculation":"Seed distribution uses top twenty positive lexical scores normalized to sum one. Iterate p=teleport*s+(1-teleport)*P^T*p, returning dangling mass to s. Combine max-normalized lexical score and walk score with blend weight. Zero lexical scores produce no seed and all scores remain zero.","selection":"Highest development family-mean MRR@10 among six nonzero settings; ties prefer lower blend then higher teleport. Graph selection is not permitted to silently choose the unchanged baseline.","controls":"Five undirected degree-preserving double-edge-swap controls, seeds 12701 through12705, ten times E attempts, no self/duplicate edges; report accepted swaps and preserved degrees. Same locked graph parameters. Degree-only baseline and empty-edge ablation; stored-directed walk exploratory."},
  "ranking": "Positive scores only. Ties by ascending checked ID. Intended item absent from positive ranking has rank null and receives zero. No false match from zero-score ties.",
  "primary": "Held-out family-mean reciprocal rank truncated at10 (MRR@10), graph minus selected lexical baseline. Three query variants are averaged within each family; eighty families are the resampling units.",
  "secondary": ["Intended-target hit@1,5,10,20", "Single-intended-target discounted rank at10; not comprehensive nDCG over all relevant concepts", "Untruncated intended-target MRR", "Query-kind and historical-band breakdowns", "Real graph versus mean of five randomized graph controls", "Hub-only and empty-edge controls", "Wins, losses, unchanged and per-query rankings", "Degree, lexical-overlap and reachability diagnostics"],
  "statistics": {"seed":127300,"clusterBootstrap":10000,"interval":"paired percentile 95% over eighty families","pairedSignFlip":10000,"test":"two-sided Monte Carlo sign-flip of family-level differences, plus-one correction","scope":"One confirmatory primary contrast. Secondary intervals and p-values descriptive/exploratory, not multiplicity-corrected. Uncertainty covers sampled synthetic families, not all future users or model behavior."},
  "interpretation": "Application and boundary queries are retrieval probes, not proof of comprehension or transfer. Any later qualitative interpretation exercise must be labeled exploratory and keep algorithm labels/target IDs hidden from its reader where possible. No transport benchmark substitutes for semantic evidence.",
  "claims": "Improvement is allowed to be absent or negative. Report every predefined method, all errors, overlap and controls. Do not edit graph, questions, targets or parameters in response to held-out outcomes. No claim of neural-embedding superiority, universal semantic search or human comprehension; dense retrieval and human relevance pools are follow-up work.",
  "references": [
    {"title":"Manning, Raghavan & Schütze — Information retrieval system evaluation","url":"https://nlp.stanford.edu/IR-book/html/htmledition/information-retrieval-system-evaluation-1.html"},
    {"title":"Manning, Raghavan & Schütze — Evaluation of ranked retrieval results","url":"https://nlp.stanford.edu/IR-book/html/htmledition/evaluation-of-ranked-retrieval-results-1.html"},
    {"title":"Manning, Raghavan & Schütze — Assessing relevance","url":"https://nlp.stanford.edu/IR-book/html/htmledition/assessing-relevance-1.html"},
    {"title":"Robertson & Zaragoza — The Probabilistic Relevance Framework: BM25 and Beyond","url":"https://www.nowpublishers.com/article/DownloadEBook/INR-019"},
    {"title":"Lofgren — Personalized PageRank Estimation and Search","url":"https://arxiv.org/abs/1507.05999"},
    {"title":"Urbano, Lima & Hanjalic — Statistical Significance Testing in Information Retrieval","url":"https://arxiv.org/abs/1905.11096"}
  ]
}