{
  "dataset": "synthetic-retrieval-eval-v1",
  "description": "24 authored paraphrases over the 12 fictional documents; separate from the four demo questions. Small diagnostic set, not a held-out benchmark.",
  "queries": 24,
  "model": "Xenova/all-MiniLM-L6-v2",
  "revision": "751bff37182d3f1213fa05d7196b954e230abad9",
  "config": {
    "bm25K1": 1.2,
    "bm25B": 0.75,
    "titleWeight": 3,
    "candidateK": 10,
    "rrfK": 60,
    "topK": 5,
    "rerank": false
  },
  "metrics": {
    "bm25": {
      "recallAt5": 0.8333333333333334,
      "mrrAt5": 0.7638888888888888
    },
    "dense": {
      "recallAt5": 1,
      "mrrAt5": 0.8958333333333334
    },
    "hybrid": {
      "recallAt5": 1,
      "mrrAt5": 0.84375
    }
  },
  "hashes": {
    "demo/documents.json": "1eb8290d5e344f693544e441d2bca8b33ed9129c2c244696679bd72e359ef477",
    "demo/queries.json": "8e357719e5242ec36fd6b3e0da73175dd5bfb943e3c967f10f48df572373e32f",
    "demo/vectors.json": "03810dfa1414064286fdfe5320c712ec3c95c3acae612142fa44492ea8d1f06e",
    "demo/eval-dataset.json": "cd0d9f0028e0e6f23f2b60b14ad0ea198858715cdf5664257638365ff78a53dc",
    "demo/eval-vectors.json": "4a9ce3dfdf3a3abd806c68d2fa7443b7d1af81eb0eb564849bb56bdbdbcfe984",
    "lib/demo-retrieval/index.ts": "65c9bc11ba262cedb29323cefa15bd097a157e82be08a9f8c0a83e6013a22a26",
    "lib/demo-retrieval/hybrid.ts": "ce3a8519211e11770b48c068ce5fc9f46d527c5664147141e4efceb61c184506",
    "scripts/evaluate-retrieval.ts": "1c405200b08b7c72625f0289045b36a7e652a47839054ac46d92ebd0ab1848b0",
    "scripts/generate-vectors.py": "914847b69c4b71317572df4e6b5d0632073a1d877db6a3cd643457f049e224ea"
  },
  "rows": [
    {
      "id": "refund-window",
      "query": "Can a first-time subscriber get their money back after two weeks?",
      "relevantChunkIds": [
        "refund-policy#1"
      ],
      "bm25": {
        "ids": [
          "refund-policy#1",
          "account-security#2",
          "data-retention#2",
          "refund-policy#2",
          "support-escalation#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "refund-policy#1",
          "subscription-policy#1",
          "subscription-policy#2",
          "customer-support#1",
          "refund-policy#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "refund-policy#1",
          "account-security#2",
          "refund-policy#2",
          "subscription-policy#1",
          "data-retention#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "refund-processing",
      "query": "Where does an approved reimbursement go and how long does it take?",
      "relevantChunkIds": [
        "refund-policy#2"
      ],
      "bm25": {
        "ids": [
          "subscription-policy#1",
          "access-control#1",
          "refund-policy#2",
          "document-governance#1",
          "finance-audit#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 0.3333333333333333
      },
      "dense": {
        "ids": [
          "refund-policy#1",
          "refund-policy#2",
          "subscription-policy#2",
          "customer-support#2",
          "subscription-policy#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 0.5
      },
      "hybrid": {
        "ids": [
          "refund-policy#2",
          "subscription-policy#1",
          "finance-audit#1",
          "refund-policy#1",
          "access-control#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "security-containment",
      "query": "Someone else has signed into my profile. What immediate containment steps apply?",
      "relevantChunkIds": [
        "account-security#1"
      ],
      "bm25": {
        "ids": [
          "employee-handbook#2",
          "incident-response#1",
          "customer-support#2",
          "support-escalation#2",
          "refund-policy#1"
        ],
        "recallAt5": 0,
        "reciprocalRankAt5": 0
      },
      "dense": {
        "ids": [
          "account-security#1",
          "account-security#2",
          "employee-handbook#2",
          "customer-support#1",
          "support-escalation#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "employee-handbook#2",
          "support-escalation#2",
          "subscription-policy#1",
          "account-security#1",
          "account-security#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 0.25
      }
    },
    {
      "id": "security-restore",
      "query": "What must happen before access is restored after a compromised login?",
      "relevantChunkIds": [
        "account-security#2"
      ],
      "bm25": {
        "ids": [
          "account-security#2",
          "employee-handbook#2",
          "employee-handbook#1",
          "refund-policy#2",
          "access-control#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "account-security#2",
          "account-security#1",
          "employee-handbook#1",
          "support-escalation#1",
          "subscription-policy#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "account-security#2",
          "employee-handbook#1",
          "access-control#2",
          "access-control#1",
          "customer-support#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "subscription-cancel",
      "query": "If I stop my subscription today, when does my access end?",
      "relevantChunkIds": [
        "subscription-policy#1"
      ],
      "bm25": {
        "ids": [
          "subscription-policy#1",
          "finance-audit#2",
          "subscription-policy#2",
          "refund-policy#1",
          "access-control#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "subscription-policy#1",
          "subscription-policy#2",
          "data-retention#1",
          "refund-policy#1",
          "access-control#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "subscription-policy#1",
          "subscription-policy#2",
          "refund-policy#1",
          "access-control#2",
          "finance-audit#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "subscription-refund",
      "query": "Does ending a subscription automatically initiate a reimbursement?",
      "relevantChunkIds": [
        "subscription-policy#2"
      ],
      "bm25": {
        "ids": [
          "subscription-policy#2",
          "subscription-policy#1",
          "refund-policy#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "subscription-policy#1",
          "subscription-policy#2",
          "refund-policy#1",
          "refund-policy#2",
          "customer-support#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 0.5
      },
      "hybrid": {
        "ids": [
          "subscription-policy#1",
          "subscription-policy#2",
          "refund-policy#1",
          "refund-policy#2",
          "customer-support#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 0.5
      }
    },
    {
      "id": "escalation-owner",
      "query": "Who communicates with the customer during a compromised-account incident?",
      "relevantChunkIds": [
        "support-escalation#1"
      ],
      "bm25": {
        "ids": [
          "customer-support#1",
          "support-escalation#1",
          "account-security#1",
          "account-security#2",
          "employee-handbook#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 0.5
      },
      "dense": {
        "ids": [
          "account-security#1",
          "support-escalation#1",
          "account-security#2",
          "customer-support#1",
          "employee-handbook#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 0.5
      },
      "hybrid": {
        "ids": [
          "account-security#1",
          "support-escalation#1",
          "customer-support#1",
          "account-security#2",
          "incident-response#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 0.5
      }
    },
    {
      "id": "escalation-details",
      "query": "What information should accompany a service interruption escalation?",
      "relevantChunkIds": [
        "support-escalation#2"
      ],
      "bm25": {
        "ids": [
          "support-escalation#2",
          "support-escalation#1",
          "employee-handbook#2",
          "incident-response#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "support-escalation#2",
          "incident-response#2",
          "incident-response#1",
          "support-escalation#1",
          "subscription-policy#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "support-escalation#2",
          "incident-response#2",
          "support-escalation#1",
          "employee-handbook#2",
          "incident-response#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "onboarding-access",
      "query": "What do new hires need to do before getting internal system access?",
      "relevantChunkIds": [
        "employee-handbook#1"
      ],
      "bm25": {
        "ids": [
          "employee-handbook#1",
          "access-control#2",
          "refund-policy#1",
          "document-governance#1",
          "access-control#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "employee-handbook#1",
          "release-checklist#1",
          "access-control#1",
          "employee-handbook#2",
          "account-security#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "employee-handbook#1",
          "access-control#1",
          "access-control#2",
          "release-checklist#1",
          "support-escalation#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "suspicious-email",
      "query": "Who should employees contact about a suspicious email?",
      "relevantChunkIds": [
        "employee-handbook#2"
      ],
      "bm25": {
        "ids": [
          "employee-handbook#2",
          "employee-handbook#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "employee-handbook#2",
          "support-escalation#1",
          "account-security#1",
          "employee-handbook#1",
          "customer-support#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "employee-handbook#2",
          "employee-handbook#1",
          "support-escalation#1",
          "account-security#1",
          "customer-support#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "support-identity",
      "query": "What should support verify before changing customer settings?",
      "relevantChunkIds": [
        "customer-support#1"
      ],
      "bm25": {
        "ids": [
          "customer-support#1",
          "customer-support#2",
          "support-escalation#1",
          "support-escalation#2",
          "data-retention#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "customer-support#1",
          "release-checklist#1",
          "account-security#2",
          "support-escalation#2",
          "incident-response#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "customer-support#1",
          "support-escalation#2",
          "customer-support#2",
          "release-checklist#1",
          "access-control#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "support-exception",
      "query": "Who handles exceptions to a customer reimbursement request?",
      "relevantChunkIds": [
        "customer-support#2"
      ],
      "bm25": {
        "ids": [
          "customer-support#2",
          "customer-support#1",
          "subscription-policy#2",
          "refund-policy#1",
          "account-security#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "customer-support#2",
          "refund-policy#2",
          "customer-support#1",
          "access-control#2",
          "subscription-policy#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "customer-support#2",
          "customer-support#1",
          "subscription-policy#2",
          "refund-policy#1",
          "account-security#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "finance-authorization",
      "query": "Which roles may inspect the complete accounting records?",
      "relevantChunkIds": [
        "access-control#1"
      ],
      "bm25": {
        "ids": [
          "employee-handbook#1",
          "refund-policy#1",
          "finance-audit#2",
          "finance-audit#1",
          "data-retention#1"
        ],
        "recallAt5": 0,
        "reciprocalRankAt5": 0
      },
      "dense": {
        "ids": [
          "finance-audit#1",
          "access-control#1",
          "access-control#2",
          "finance-audit#2",
          "data-retention#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 0.5
      },
      "hybrid": {
        "ids": [
          "finance-audit#1",
          "finance-audit#2",
          "employee-handbook#1",
          "access-control#1",
          "access-control#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 0.25
      }
    },
    {
      "id": "support-finance-limits",
      "query": "Can a support agent see unredacted accounting details?",
      "relevantChunkIds": [
        "access-control#2"
      ],
      "bm25": {
        "ids": [
          "customer-support#1",
          "support-escalation#1",
          "customer-support#2",
          "support-escalation#2",
          "data-retention#1"
        ],
        "recallAt5": 0,
        "reciprocalRankAt5": 0
      },
      "dense": {
        "ids": [
          "access-control#2",
          "finance-audit#2",
          "access-control#1",
          "finance-audit#1",
          "customer-support#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "customer-support#1",
          "access-control#2",
          "support-escalation#1",
          "customer-support#2",
          "account-security#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 0.5
      }
    },
    {
      "id": "old-document",
      "query": "Are obsolete policy revisions deleted from the audit history?",
      "relevantChunkIds": [
        "document-governance#1"
      ],
      "bm25": {
        "ids": [
          "document-governance#1",
          "access-control#1",
          "data-retention#2",
          "finance-audit#2",
          "document-governance#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "document-governance#1",
          "data-retention#2",
          "account-security#2",
          "finance-audit#2",
          "access-control#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "document-governance#1",
          "data-retention#2",
          "finance-audit#2",
          "access-control#1",
          "document-governance#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "citation-refresh",
      "query": "What must happen to answers that cite an outdated policy revision?",
      "relevantChunkIds": [
        "document-governance#2"
      ],
      "bm25": {
        "ids": [
          "employee-handbook#2",
          "document-governance#2",
          "subscription-policy#2",
          "account-security#1",
          "data-retention#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 0.5
      },
      "dense": {
        "ids": [
          "document-governance#2",
          "document-governance#1",
          "subscription-policy#1",
          "account-security#2",
          "access-control#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "document-governance#2",
          "data-retention#1",
          "data-retention#2",
          "access-control#2",
          "access-control#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "incident-coordinator",
      "query": "Who is responsible for coordinating incident containment and recovery?",
      "relevantChunkIds": [
        "incident-response#1"
      ],
      "bm25": {
        "ids": [
          "incident-response#1",
          "incident-response#2",
          "account-security#1",
          "support-escalation#1",
          "account-security#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "incident-response#1",
          "incident-response#2",
          "support-escalation#1",
          "support-escalation#2",
          "account-security#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "incident-response#1",
          "incident-response#2",
          "support-escalation#1",
          "account-security#1",
          "account-security#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "incident-learning",
      "query": "What does a post-incident review need to record?",
      "relevantChunkIds": [
        "incident-response#2"
      ],
      "bm25": {
        "ids": [
          "incident-response#2",
          "incident-response#1",
          "support-escalation#1",
          "access-control#2",
          "release-checklist#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "incident-response#2",
          "incident-response#1",
          "finance-audit#1",
          "data-retention#2",
          "account-security#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "incident-response#2",
          "incident-response#1",
          "support-escalation#2",
          "access-control#2",
          "account-security#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "ticket-retention",
      "query": "How long does this fictional workspace keep support tickets?",
      "relevantChunkIds": [
        "data-retention#1"
      ],
      "bm25": {
        "ids": [
          "data-retention#1",
          "data-retention#2",
          "support-escalation#1",
          "customer-support#1",
          "customer-support#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "data-retention#1",
          "subscription-policy#1",
          "support-escalation#2",
          "refund-policy#1",
          "data-retention#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "data-retention#1",
          "data-retention#2",
          "support-escalation#2",
          "access-control#2",
          "subscription-policy#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "audit-retention",
      "query": "Do audit events have the same retention schedule as support tickets?",
      "relevantChunkIds": [
        "data-retention#2"
      ],
      "bm25": {
        "ids": [
          "data-retention#2",
          "data-retention#1",
          "support-escalation#1",
          "finance-audit#2",
          "finance-audit#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "data-retention#2",
          "data-retention#1",
          "finance-audit#2",
          "finance-audit#1",
          "subscription-policy#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "data-retention#2",
          "data-retention#1",
          "finance-audit#2",
          "finance-audit#1",
          "support-escalation#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "audit-export",
      "query": "How is an auditor’s financial export authorized and monitored?",
      "relevantChunkIds": [
        "finance-audit#1"
      ],
      "bm25": {
        "ids": [
          "finance-audit#1",
          "access-control#1",
          "finance-audit#2",
          "access-control#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "finance-audit#1",
          "access-control#1",
          "finance-audit#2",
          "access-control#2",
          "data-retention#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "finance-audit#1",
          "access-control#1",
          "finance-audit#2",
          "access-control#2",
          "data-retention#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "audit-expiry",
      "query": "When is temporary auditor access removed?",
      "relevantChunkIds": [
        "finance-audit#2"
      ],
      "bm25": {
        "ids": [
          "access-control#2",
          "data-retention#1",
          "access-control#1",
          "employee-handbook#1",
          "data-retention#2"
        ],
        "recallAt5": 0,
        "reciprocalRankAt5": 0
      },
      "dense": {
        "ids": [
          "finance-audit#1",
          "finance-audit#2",
          "data-retention#2",
          "access-control#2",
          "access-control#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 0.5
      },
      "hybrid": {
        "ids": [
          "access-control#2",
          "finance-audit#1",
          "data-retention#1",
          "finance-audit#2",
          "access-control#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 0.25
      }
    },
    {
      "id": "release-baseline",
      "query": "Which identities and checks should be fixed before a release?",
      "relevantChunkIds": [
        "release-checklist#1"
      ],
      "bm25": {
        "ids": [
          "release-checklist#1",
          "release-checklist#2",
          "incident-response#2",
          "employee-handbook#1",
          "support-escalation#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "release-checklist#1",
          "release-checklist#2",
          "customer-support#1",
          "account-security#2",
          "document-governance#1"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "release-checklist#1",
          "release-checklist#2",
          "customer-support#1",
          "incident-response#2",
          "support-escalation#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    },
    {
      "id": "release-proof",
      "query": "Does a successful local check demonstrate a production deployment?",
      "relevantChunkIds": [
        "release-checklist#2"
      ],
      "bm25": {
        "ids": [
          "release-checklist#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "dense": {
        "ids": [
          "release-checklist#2",
          "release-checklist#1",
          "incident-response#2",
          "finance-audit#1",
          "employee-handbook#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      },
      "hybrid": {
        "ids": [
          "release-checklist#2",
          "release-checklist#1",
          "incident-response#2",
          "finance-audit#1",
          "employee-handbook#2"
        ],
        "recallAt5": 1,
        "reciprocalRankAt5": 1
      }
    }
  ],
  "generatedAt": "2026-09-18T16:06:49.178Z"
}