{
  "recognizer": "Jarvis Parakeet TDT CTC 110m INT8, no hotwords",
  "summary": {
    "cpp-alba3-holdout": {
      "words": 504,
      "errors": 13,
      "wer": 0.025793650793650792,
      "shortTranscripts": [
        "Done.",
        "Done.",
        "Done."
      ]
    },
    "cpp-expresso-cached": {
      "words": 460,
      "errors": 24,
      "wer": 0.05217391304347826,
      "shortTranscripts": [
        "done",
        "Done.",
        "done.",
        "Don.",
        "Done."
      ]
    },
    "cpp-expresso-holdout": {
      "words": 504,
      "errors": 12,
      "wer": 0.023809523809523808,
      "shortTranscripts": [
        "Done.",
        "done.",
        "done."
      ]
    },
    "kitten-holdout": {
      "words": 504,
      "errors": 0,
      "wer": 0.0,
      "shortTranscripts": [
        "Done.",
        "Done.",
        "Done."
      ]
    }
  },
  "records": [
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r0",
      "scenario": "ambiguous",
      "transcript": "I found two tasks named login, choose the desktop task or the mobile task.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r0",
      "scenario": "cancel",
      "transcript": "canceled.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 1,
      "substitutions": 1,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 1.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r0",
      "scenario": "correction",
      "transcript": "I have not pushed the changes. The tests passed, but the build failed.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r0",
      "scenario": "disconnected",
      "transcript": "Your remote node is disconnected. I cannot start that task until it reconnects.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r0",
      "scenario": "long",
      "transcript": "The connection dropped while the provider was working, so I have kept the task on its original node and will show its durable result when that node reconnects, while the other projects remain available and the selected tasks stays unchanged until you choose a different one.",
      "referenceWords": 46,
      "hypothesisWords": 46,
      "errors": 1,
      "substitutions": 1,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.021739130434782608
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r0",
      "scenario": "names",
      "transcript": "Jarvis is ready. The Codex task is waiting for approval.",
      "referenceWords": 10,
      "hypothesisWords": 10,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r0",
      "scenario": "no",
      "transcript": "No.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r0",
      "scenario": "numbers",
      "transcript": "Three checks passed, and two failed. The worker used five hundred megabytes of memory.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r0",
      "scenario": "punctuation",
      "transcript": "Don't restart the server. It's still running. Review the changes first.",
      "referenceWords": 11,
      "hypothesisWords": 11,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r0",
      "scenario": "question",
      "transcript": "Which task did you mean?",
      "referenceWords": 5,
      "hypothesisWords": 5,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r0",
      "scenario": "reverse",
      "transcript": "Voice is disabled. Enable it again in settings when you want spoken reports.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r0",
      "scenario": "short",
      "transcript": "Done.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r0",
      "scenario": "stop",
      "transcript": "Stopping.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r0",
      "scenario": "typical",
      "transcript": "The task is complete. I updated the login flow and added regression tests, all targeted checks passed. You can review the changes in the workspace.",
      "referenceWords": 25,
      "hypothesisWords": 25,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r1",
      "scenario": "ambiguous",
      "transcript": "I found two tasks named login. Choose the desktop task or the mobile task.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r1",
      "scenario": "cancel",
      "transcript": "canceled.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 1,
      "substitutions": 1,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 1.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r1",
      "scenario": "correction",
      "transcript": "I have not pushed the changes. The test passed, but the bill failed.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 2,
      "substitutions": 2,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.15384615384615385
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r1",
      "scenario": "disconnected",
      "transcript": "Your remote node is disconnected. I cannot start that task until it reconnects.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r1",
      "scenario": "long",
      "transcript": "The connection dropped while the provider was working, so I have kept the task on its original node, and will show its durable result when that node reconnects, while the other projects remain available, and the selected tasks until you choose a different one.",
      "referenceWords": 46,
      "hypothesisWords": 44,
      "errors": 3,
      "substitutions": 1,
      "deletions": 2,
      "insertions": 0,
      "wordErrorRate": 0.06521739130434782
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r1",
      "scenario": "names",
      "transcript": "Jarvis is ready. The Codex task is waiting for approval.",
      "referenceWords": 10,
      "hypothesisWords": 10,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r1",
      "scenario": "no",
      "transcript": "No.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r1",
      "scenario": "numbers",
      "transcript": "Three checks passed and two failed. The worker used five hundred megabytes of memory.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r1",
      "scenario": "punctuation",
      "transcript": "Don't restart the server. It's still running. Review the changes first.",
      "referenceWords": 11,
      "hypothesisWords": 11,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r1",
      "scenario": "question",
      "transcript": "Which task did you mean?",
      "referenceWords": 5,
      "hypothesisWords": 5,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r1",
      "scenario": "reverse",
      "transcript": "Voice is disabled. Enable it again in settings when you want spoken reports.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r1",
      "scenario": "short",
      "transcript": "Done.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r1",
      "scenario": "stop",
      "transcript": "Stopping.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r1",
      "scenario": "typical",
      "transcript": "The task is complete. I updated the login flow and added regression tests. All targeted checks passed. You can review the changes in the workspace.",
      "referenceWords": 25,
      "hypothesisWords": 25,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r2",
      "scenario": "ambiguous",
      "transcript": "I found two tasks named login, choose the desktop task or the mobile task.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r2",
      "scenario": "cancel",
      "transcript": "canceled.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 1,
      "substitutions": 1,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 1.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r2",
      "scenario": "correction",
      "transcript": "I have not pushed the changes. The tests passed, but the build failed.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r2",
      "scenario": "disconnected",
      "transcript": "Your remote node is disconnected. I cannot start that task until it reconnects.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r2",
      "scenario": "long",
      "transcript": "The connection dropped while the provider was working, so I have kept the task on its original node and will show its durable result when that node reconnects while the other projects remain available and the selected tasks until you choose a different one.",
      "referenceWords": 46,
      "hypothesisWords": 44,
      "errors": 3,
      "substitutions": 1,
      "deletions": 2,
      "insertions": 0,
      "wordErrorRate": 0.06521739130434782
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r2",
      "scenario": "names",
      "transcript": "Jarvis is ready. The Kodex task is waiting for approval.",
      "referenceWords": 10,
      "hypothesisWords": 10,
      "errors": 1,
      "substitutions": 1,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.1
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r2",
      "scenario": "no",
      "transcript": "No.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r2",
      "scenario": "numbers",
      "transcript": "Three checks passed and two failed. The worker used five hundred megabytes of memory.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r2",
      "scenario": "punctuation",
      "transcript": "Don't restart the server. It's still running. Review the changes first.",
      "referenceWords": 11,
      "hypothesisWords": 11,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r2",
      "scenario": "question",
      "transcript": "Which task did you mean?",
      "referenceWords": 5,
      "hypothesisWords": 5,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r2",
      "scenario": "reverse",
      "transcript": "Voice is disabled. Enable it again in settings when you want spoken reports.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r2",
      "scenario": "short",
      "transcript": "Done.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r2",
      "scenario": "stop",
      "transcript": "Stopping.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-alba3-holdout",
      "repetition": "r2",
      "scenario": "typical",
      "transcript": "The task is complete. I updated the login flow and added regression tests. All targeted checks passed. You can review the changes in the workspace.",
      "referenceWords": 25,
      "hypothesisWords": 25,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r0",
      "scenario": "long",
      "transcript": "I finished the requested update and check the paths that share this behavior. The desktop client now keeps the selected tasks stable across reconnects while the mobile client reports a disconnected node instead of silently moving work elsewhere. Focused tests cover local and remote routing, cancellation and the reverse transition. The targeted type check and test files pass and no live user data was opened or changed.",
      "referenceWords": 66,
      "hypothesisWords": 67,
      "errors": 4,
      "substitutions": 3,
      "deletions": 0,
      "insertions": 1,
      "wordErrorRate": 0.06060606060606061
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r0",
      "scenario": "short",
      "transcript": "done",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r0",
      "scenario": "typical",
      "transcript": "The task is complete. I updated the login flow and added regression tests. All targeted checks passed, you can review the changes in the workspace.",
      "referenceWords": 25,
      "hypothesisWords": 25,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r1",
      "scenario": "long",
      "transcript": "I finished the requested update and check the paths that share this behavior. The desktop client now keeps the selected tasks stable across reconnects while the mobile client reports a disconnected node instead of silently moving work elsewhere. Focus tests cover local and remote routing, cancellation and the reverse transition. The targeted type check and test files pass and no live user data was opened or changed.",
      "referenceWords": 66,
      "hypothesisWords": 67,
      "errors": 5,
      "substitutions": 4,
      "deletions": 0,
      "insertions": 1,
      "wordErrorRate": 0.07575757575757576
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r1",
      "scenario": "short",
      "transcript": "Done.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r1",
      "scenario": "typical",
      "transcript": "The task is complete. I updated the login flow and added regression tests. All targeted checks passed, you can review the changes in the workspace.",
      "referenceWords": 25,
      "hypothesisWords": 25,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r2",
      "scenario": "long",
      "transcript": "I finished the requested update and check the paths that share this behavior. The desktop client now keeps the selected tasks stable across reconnects while the mobile client reports a disconnected node instead of silently moving work elsewhere. Focused tests cover local and remote routing, cancellation, and the reverse transition. The targeted type check and test files pass and no live user data was opened or changed.",
      "referenceWords": 66,
      "hypothesisWords": 67,
      "errors": 4,
      "substitutions": 3,
      "deletions": 0,
      "insertions": 1,
      "wordErrorRate": 0.06060606060606061
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r2",
      "scenario": "short",
      "transcript": "done.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r2",
      "scenario": "typical",
      "transcript": "The task is complete. I updated the login flow and added regression tests, all targeted checks passed. You can review the changes in the workspace.",
      "referenceWords": 25,
      "hypothesisWords": 25,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r3",
      "scenario": "long",
      "transcript": "I finished the requested update and check the paths that share this behavior. The desktop client now keeps the selected tasks stable across reconnects while the mobile client reports a disconnected node instead of silently moving work elsewhere. Focus tests cover local and remote routing, cancellation and the reverse transition. The targeted type check and test files pass and no live user data was opened or changed.",
      "referenceWords": 66,
      "hypothesisWords": 67,
      "errors": 5,
      "substitutions": 4,
      "deletions": 0,
      "insertions": 1,
      "wordErrorRate": 0.07575757575757576
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r3",
      "scenario": "short",
      "transcript": "Don.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 1,
      "substitutions": 1,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 1.0
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r3",
      "scenario": "typical",
      "transcript": "The task is complete. I updated the login flow and added regression tests. All targeted checks passed. You can review the changes in the workspace.",
      "referenceWords": 25,
      "hypothesisWords": 25,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r4",
      "scenario": "long",
      "transcript": "I finished the requested update and check the paths that share this behavior. The desktop client now keeps the selected tasks stable across reconnects, while the mobile client reports a disconnected node instead of silently moving work elsewhere. Focus tests cover local and remote routing, cancellation and the reverse transition. The targeted type check and test files pass and no live user data was opened or changed.",
      "referenceWords": 66,
      "hypothesisWords": 67,
      "errors": 5,
      "substitutions": 4,
      "deletions": 0,
      "insertions": 1,
      "wordErrorRate": 0.07575757575757576
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r4",
      "scenario": "short",
      "transcript": "Done.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-cached",
      "repetition": "r4",
      "scenario": "typical",
      "transcript": "The task is complete. I updated the login flow and added regression tests, all targeted checks passed, you can review the changes in the workspace.",
      "referenceWords": 25,
      "hypothesisWords": 25,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r0",
      "scenario": "ambiguous",
      "transcript": "I found two tasks named log in. Choose the desktop task or the mobile task.",
      "referenceWords": 14,
      "hypothesisWords": 15,
      "errors": 2,
      "substitutions": 1,
      "deletions": 0,
      "insertions": 1,
      "wordErrorRate": 0.14285714285714285
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r0",
      "scenario": "cancel",
      "transcript": "canceled.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 1,
      "substitutions": 1,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 1.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r0",
      "scenario": "correction",
      "transcript": "I have not pushed the changes. The test passed but the build failed.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 1,
      "substitutions": 1,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.07692307692307693
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r0",
      "scenario": "disconnected",
      "transcript": "your remote node is disconnected. I cannot start that task until it reconnects.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r0",
      "scenario": "long",
      "transcript": "The connection dropped while the provider was working, so I've kept the task on its original node and will show its durable result when that node reconnects, while the other projects remain available and the selected task stays unchanged until you choose a different one.",
      "referenceWords": 46,
      "hypothesisWords": 45,
      "errors": 2,
      "substitutions": 1,
      "deletions": 1,
      "insertions": 0,
      "wordErrorRate": 0.043478260869565216
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r0",
      "scenario": "names",
      "transcript": "Jarvis is ready. The Codex task is waiting for approval.",
      "referenceWords": 10,
      "hypothesisWords": 10,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r0",
      "scenario": "no",
      "transcript": "No.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r0",
      "scenario": "numbers",
      "transcript": "Three checks passed and two failed. The worker used five hundred megabytes of memory.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r0",
      "scenario": "punctuation",
      "transcript": "Don't restart the server. It's still running. Review the changes first.",
      "referenceWords": 11,
      "hypothesisWords": 11,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r0",
      "scenario": "question",
      "transcript": "Which task did you mean?",
      "referenceWords": 5,
      "hypothesisWords": 5,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r0",
      "scenario": "reverse",
      "transcript": "Voice is disabled. Enable it again in settings when you want spoken reports.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r0",
      "scenario": "short",
      "transcript": "Done.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r0",
      "scenario": "stop",
      "transcript": "stopping.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r0",
      "scenario": "typical",
      "transcript": "The task is complete. I updated the login flow and added regression tests. All targeted checks passed. You can review the changes in the workspace.",
      "referenceWords": 25,
      "hypothesisWords": 25,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r1",
      "scenario": "ambiguous",
      "transcript": "I found two tasks named login. Choose the desktop task or the mobile task.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r1",
      "scenario": "cancel",
      "transcript": "cancelled",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r1",
      "scenario": "correction",
      "transcript": "I have not pushed the changes. The test passed, but the bill failed.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 2,
      "substitutions": 2,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.15384615384615385
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r1",
      "scenario": "disconnected",
      "transcript": "Your remote node is disconnected. I cannot start that task until it reconnects.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r1",
      "scenario": "long",
      "transcript": "The connection dropped while the provider was working, so I have kept the task on its original node and will show its durable result when that node reconnects while the other projects remain available and the selected task stays unchanged until you choose a different one.",
      "referenceWords": 46,
      "hypothesisWords": 46,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r1",
      "scenario": "names",
      "transcript": "Jarvis is ready. The Kodak's task is waiting for approval.",
      "referenceWords": 10,
      "hypothesisWords": 10,
      "errors": 1,
      "substitutions": 1,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.1
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r1",
      "scenario": "no",
      "transcript": "No.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r1",
      "scenario": "numbers",
      "transcript": "Three checks passed and two failed. The worker used five hundred megabytes of memory.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r1",
      "scenario": "punctuation",
      "transcript": "Don't restart the server. It's still running. Review the changes first.",
      "referenceWords": 11,
      "hypothesisWords": 11,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r1",
      "scenario": "question",
      "transcript": "Which task did you mean?",
      "referenceWords": 5,
      "hypothesisWords": 5,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r1",
      "scenario": "reverse",
      "transcript": "voice is disabled. Enable it again in settings when you want spoken reports.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r1",
      "scenario": "short",
      "transcript": "done.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r1",
      "scenario": "stop",
      "transcript": "stopping",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r1",
      "scenario": "typical",
      "transcript": "The task is complete. I updated the login flow and added regression tests. All targeted checks passed. You can review the changes in the workspace.",
      "referenceWords": 25,
      "hypothesisWords": 25,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r2",
      "scenario": "ambiguous",
      "transcript": "I found two tasks named login, choose the desktop task or the mobile task.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r2",
      "scenario": "cancel",
      "transcript": "canceled.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 1,
      "substitutions": 1,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 1.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r2",
      "scenario": "correction",
      "transcript": "I have not pushed the changes. The test passed, but the bill failed.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 2,
      "substitutions": 2,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.15384615384615385
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r2",
      "scenario": "disconnected",
      "transcript": "Your remote node is disconnected. I cannot start that task until it reconnects.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r2",
      "scenario": "long",
      "transcript": "The connection dropped while the provider was working, so I have kept the task on its original node and will show its durable result when that node reconnects while the other projects remain available and the selected task stays unchanged until you choose a different one.",
      "referenceWords": 46,
      "hypothesisWords": 46,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r2",
      "scenario": "names",
      "transcript": "Jarvis is ready. The Codex task is waiting for approval.",
      "referenceWords": 10,
      "hypothesisWords": 10,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r2",
      "scenario": "no",
      "transcript": "No.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r2",
      "scenario": "numbers",
      "transcript": "Three checks passed and two failed, the worker used five hundred megabytes of memory.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r2",
      "scenario": "punctuation",
      "transcript": "Don't restart the server. It's still running. Review the changes first.",
      "referenceWords": 11,
      "hypothesisWords": 11,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r2",
      "scenario": "question",
      "transcript": "Which task did you mean?",
      "referenceWords": 5,
      "hypothesisWords": 5,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r2",
      "scenario": "reverse",
      "transcript": "Voice is disabled. Enable it again in settings when you want spoken reports.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r2",
      "scenario": "short",
      "transcript": "done.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r2",
      "scenario": "stop",
      "transcript": "Stopping.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "cpp-expresso-holdout",
      "repetition": "r2",
      "scenario": "typical",
      "transcript": "The task is complete. I updated the login flow and added regression tests, all targeted checks passed, you can review the changes in the workspace.",
      "referenceWords": 25,
      "hypothesisWords": 25,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r0",
      "scenario": "ambiguous",
      "transcript": "I found two tasks named login. Choose the desktop task or the mobile task.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r0",
      "scenario": "cancel",
      "transcript": "Cancelled.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r0",
      "scenario": "correction",
      "transcript": "I have not pushed the changes, the tests passed, but the build failed.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r0",
      "scenario": "disconnected",
      "transcript": "Your remote node is disconnected. I cannot start that task until it reconnects.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r0",
      "scenario": "long",
      "transcript": "The connection dropped while the provider was working, so I have kept the task on its original node and will show its durable result when that node reconnects, while the other projects remain available and the selected task stays unchanged until you choose a different one.",
      "referenceWords": 46,
      "hypothesisWords": 46,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r0",
      "scenario": "names",
      "transcript": "Jarvis is ready. The codex task is waiting for approval.",
      "referenceWords": 10,
      "hypothesisWords": 10,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r0",
      "scenario": "no",
      "transcript": "No.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r0",
      "scenario": "numbers",
      "transcript": "Three checks passed and two failed. The worker used five hundred megabytes of memory.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r0",
      "scenario": "punctuation",
      "transcript": "Don't restart the server, it's still running. Review the changes first.",
      "referenceWords": 11,
      "hypothesisWords": 11,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r0",
      "scenario": "question",
      "transcript": "Which task did you mean?",
      "referenceWords": 5,
      "hypothesisWords": 5,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r0",
      "scenario": "reverse",
      "transcript": "Voice is disabled. Enable it again in settings when you want spoken reports.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r0",
      "scenario": "short",
      "transcript": "Done.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r0",
      "scenario": "stop",
      "transcript": "Stopping.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r0",
      "scenario": "typical",
      "transcript": "The task is complete. I updated the login flow and added regression tests. All targeted checks passed. You can review the changes in the workspace.",
      "referenceWords": 25,
      "hypothesisWords": 25,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r1",
      "scenario": "ambiguous",
      "transcript": "I found two tasks named login. Choose the desktop task or the mobile task.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r1",
      "scenario": "cancel",
      "transcript": "Cancelled.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r1",
      "scenario": "correction",
      "transcript": "I have not pushed the changes, the tests passed, but the build failed.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r1",
      "scenario": "disconnected",
      "transcript": "Your remote node is disconnected. I cannot start that task until it reconnects.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r1",
      "scenario": "long",
      "transcript": "The connection dropped while the provider was working, so I have kept the task on its original node and will show its durable result when that node reconnects, while the other projects remain available and the selected task stays unchanged until you choose a different one.",
      "referenceWords": 46,
      "hypothesisWords": 46,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r1",
      "scenario": "names",
      "transcript": "Jarvis is ready. The codex task is waiting for approval.",
      "referenceWords": 10,
      "hypothesisWords": 10,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r1",
      "scenario": "no",
      "transcript": "No.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r1",
      "scenario": "numbers",
      "transcript": "Three checks passed and two failed. The worker used five hundred megabytes of memory.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r1",
      "scenario": "punctuation",
      "transcript": "Don't restart the server, it's still running. Review the changes first.",
      "referenceWords": 11,
      "hypothesisWords": 11,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r1",
      "scenario": "question",
      "transcript": "Which task did you mean?",
      "referenceWords": 5,
      "hypothesisWords": 5,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r1",
      "scenario": "reverse",
      "transcript": "Voice is disabled. Enable it again in settings when you want spoken reports.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r1",
      "scenario": "short",
      "transcript": "Done.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r1",
      "scenario": "stop",
      "transcript": "Stopping.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r1",
      "scenario": "typical",
      "transcript": "The task is complete. I updated the login flow and added regression tests. All targeted checks passed. You can review the changes in the workspace.",
      "referenceWords": 25,
      "hypothesisWords": 25,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r2",
      "scenario": "ambiguous",
      "transcript": "I found two tasks named login. Choose the desktop task or the mobile task.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r2",
      "scenario": "cancel",
      "transcript": "Cancelled.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r2",
      "scenario": "correction",
      "transcript": "I have not pushed the changes, the tests passed, but the build failed.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r2",
      "scenario": "disconnected",
      "transcript": "Your remote node is disconnected. I cannot start that task until it reconnects.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r2",
      "scenario": "long",
      "transcript": "The connection dropped while the provider was working, so I have kept the task on its original node and will show its durable result when that node reconnects while the other projects remain available and the selected task stays unchanged until you choose a different one.",
      "referenceWords": 46,
      "hypothesisWords": 46,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r2",
      "scenario": "names",
      "transcript": "Jarvis is ready. The codex task is waiting for approval.",
      "referenceWords": 10,
      "hypothesisWords": 10,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r2",
      "scenario": "no",
      "transcript": "No.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r2",
      "scenario": "numbers",
      "transcript": "Three checks passed and two failed. The worker used five hundred megabytes of memory.",
      "referenceWords": 14,
      "hypothesisWords": 14,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r2",
      "scenario": "punctuation",
      "transcript": "Don't restart the server, it's still running. Review the changes first.",
      "referenceWords": 11,
      "hypothesisWords": 11,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r2",
      "scenario": "question",
      "transcript": "Which task did you mean?",
      "referenceWords": 5,
      "hypothesisWords": 5,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r2",
      "scenario": "reverse",
      "transcript": "Voice is disabled. Enable it again in settings when you want spoken reports.",
      "referenceWords": 13,
      "hypothesisWords": 13,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r2",
      "scenario": "short",
      "transcript": "Done.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r2",
      "scenario": "stop",
      "transcript": "Stopping.",
      "referenceWords": 1,
      "hypothesisWords": 1,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    },
    {
      "engine": "kitten-holdout",
      "repetition": "r2",
      "scenario": "typical",
      "transcript": "The task is complete. I updated the login flow and added regression tests. All targeted checks passed. You can review the changes in the workspace.",
      "referenceWords": 25,
      "hypothesisWords": 25,
      "errors": 0,
      "substitutions": 0,
      "deletions": 0,
      "insertions": 0,
      "wordErrorRate": 0.0
    }
  ]
}