diff --git a/README.md b/README.md index b71ec0a..35287ef 100644 --- a/README.md +++ b/README.md @@ -46,35 +46,44 @@ with translated dependency references, materialized multi-ref annotations, pinne classes with retrieval completeness exposed on the frame, and a single canonical `entityRefKey` exported from core and used by kernel, resolver, and context alike. -`@circe/perception-jev` now provides an injected client interface and a read-only evaluation -loop. A host must supply the live SDK/transport implementation. No live model integration -or held-out semantic accuracy claim is included. - -Durability now covers both the kernel and the semantic environment. `@circe/journal-file` -reconstructs kernel state; `@circe/context-file` persists observations, relations, events, -assertions, annotations, bindings, decision traces and context frames as an append-only -operation log and replays it deterministically on open. `@circe/file-log` is the shared -append-boundary primitive. `@circe/inspector` hosts a continuing interaction: one runtime, -journal and semantic store across turns, including restarts, instead of a fresh store per -run. The one-shot `runInteractionPipeline` is a thin wrapper over that host. - -The held-out evaluation harness (`apps/inspector/src/heldout.ts`, run with -`pnpm --filter @circe/inspector eval:heldout`) drives recorded reference-model transcripts -through the real pipeline and scores retrieval, policy, compiled commands, effects and -restart recovery. It measures the pipeline, not language understanding: the recorded client -can be replaced by a live transport, but no live model integration or held-out semantic -accuracy claim is included, and risk thresholds remain initial policy values, not -outcome-calibrated guarantees. The semantic RED corpus still records missing interpretation -behavior; tests asserting that it remains RED must not be counted as successful semantic -understanding. - -Run `pnpm typecheck`, `pnpm test`, and `pnpm --filter @circe/inspector start` to check the -implemented contracts and scripted demo. `pnpm --filter @circe/inspector demo:continuing` -runs two utterances across a real restart, and -`pnpm --filter @circe/inspector eval:heldout` runs the recorded held-out corpus. -`node tools/anti-corruption/check.mjs` checks package boundaries. Multi-process -coordination and log compaction remain deferred for both file logs; they are suitable for -their current single-process host contract. +`@circe/perception-jev` provides an injected client interface and a native-fetch +TypeSafe client, `createTypeSafeJevClient`, using `/v1/systemone`. The transport maps +bounded binary, choice and score questions to the provider API and validates the returned +answers through the shared decoder. Mocked HTTP tests verify the transport contract; +they do not measure model accuracy. + +Durability covers kernel state and semantic context. `@circe/journal-file` reconstructs +commands and effects; `@circe/context-file` replays observations, relations, events, +assertions, annotations, bindings, decision traces and frames. `@circe/file-log` supplies +the shared append and tail-repair implementation. + +The inspector's continuing host serializes observations, turns, typed commands and +reconciliation. A third file records the first interpretation of each observation before +commands can run. Retrying that observation reuses the recorded interpretation, including +abstention, and rejects a changed payload. Decision traces are persisted before dispatch. +The host exposes `command` for typed confirmation, revision, cancellation and resumption, +and `reconcile` for unknown effects. Natural-language controls still require an application +workflow that translates accepted decisions into those typed commands. + +`pnpm --filter @circe/inspector eval:heldout` runs hand-written answer fixtures through +the real pipeline. It checks retrieval, policy, compiled commands, successful execution, +actual target invocation and restart recovery. These fixtures test contracts, not language +understanding. Each evaluation uses fresh files even when an output directory is supplied. + +`pnpm --filter @circe/inspector eval:live` uses the TypeSafe transport on a small synthetic +corpus covering requests, a non-mutating query, exclusion and restart. Set `TYPESAFE_API_KEY` +and `TYPESAFE_MODEL` to an account-supported model; the command fails without them. Set +`CIRCE_EVAL_DIRECTORY` to preserve logs and `live-evaluation.json`. Only local test executors +run. The command sends the synthetic frames to TypeSafe. See +[the client documentation](packages/perception-jev/README.md) for transport configuration. +No live results are checked in yet. This corpus does not establish broad semantic accuracy +or calibrate the initial risk thresholds. The 18 semantic RED cases remain a record of +missing deterministic interpretation, not successful language understanding. + +Run `pnpm typecheck`, `pnpm test`, and `pnpm --filter @circe/inspector start` for the contract +suite and scripted demo. `pnpm --filter @circe/inspector demo:continuing` runs two utterances +across a restart. `node tools/anti-corruption/check.mjs` checks package boundaries. +One process owns each store path. Multi-process coordination and log compaction are deferred. - `ARCHITECTURE.md` — types, boundaries, state machines, eval plan, decision log. **Start here for the kernel.** - `docs/SEMANTIC-BOUNDARY.md` — what the deterministic resolver may and may not do (frozen). diff --git a/apps/inspector/package.json b/apps/inspector/package.json index fe8f449..3b42b38 100644 --- a/apps/inspector/package.json +++ b/apps/inspector/package.json @@ -8,7 +8,8 @@ "demo:continuing": "tsx src/continuing-cli.ts", "eval:heldout": "tsx src/heldout-cli.ts", "typecheck": "tsc --noEmit", - "test": "vitest run" + "test": "vitest run", + "eval:live": "tsx src/live-cli.ts" }, "dependencies": { "@circe/context": "workspace:*", @@ -17,7 +18,8 @@ "@circe/evals": "workspace:*", "@circe/journal-file": "workspace:*", "@circe/perception": "workspace:*", - "@circe/perception-jev": "workspace:*" + "@circe/perception-jev": "workspace:*", + "@circe/file-log": "workspace:*" }, "devDependencies": { "@types/node": "^24.0.0", diff --git a/apps/inspector/src/file-host.ts b/apps/inspector/src/file-host.ts index 8b03417..09e82cb 100644 --- a/apps/inspector/src/file-host.ts +++ b/apps/inspector/src/file-host.ts @@ -1,5 +1,6 @@ import { createFileContextStore } from "@circe/context-file"; import { createFileJournal } from "@circe/journal-file"; +import { createFileAcceptedTurnStore } from "./turn-store"; import { createContinuingInteractionHost, type ContinuingInteractionConfig, @@ -7,16 +8,18 @@ import { } from "./host"; /** - * File-backed continuing interaction host: the journal and the semantic store - * both live on disk, so a restart reconstructs the same runtime and the same + * File-backed continuing interaction host: the journal, semantic store and accepted turns + * live on disk, so a restart reconstructs the same runtime and the same * semantic history. */ export interface FileContinuingInteractionConfig - extends Omit { + extends Omit { journalPath: string; contextPath: string; - /** fsync appends for both files (default true) */ + /** durable accepted-interpretation log (defaults to `${contextPath}.turns.jsonl`) */ + turnPath?: string; + /** fsync appends for all three files (default true) */ durable?: boolean; } @@ -29,25 +32,58 @@ export function createFileContinuingInteractionHost( ): FileContinuingInteractionHost { const options = config.durable !== undefined ? { durable: config.durable } : {}; const journal = createFileJournal(config.journalPath, options); - const context = createFileContextStore(config.contextPath, options); - const host = createContinuingInteractionHost({ - interactionId: config.interactionId, - context, - journal, - capabilities: config.capabilities, - workflow: config.workflow, - backend: config.backend, - policy: config.policy, - propositions: config.propositions, - authorities: config.authorities, - executors: config.executors, - ...(config.beforeCommands !== undefined ? { beforeCommands: config.beforeCommands } : {}), - }); - return { - ...host, - close(): void { - context.close(); - journal.close(); - }, - }; + try { + const context = createFileContextStore(config.contextPath, options); + try { + const turnStore = createFileAcceptedTurnStore(config.turnPath ?? `${config.contextPath}.turns.jsonl`, options); + try { + const host = createContinuingInteractionHost({ + interactionId: config.interactionId, + context, + journal, + turnStore, + capabilities: config.capabilities, + workflow: config.workflow, + backend: config.backend, + policy: config.policy, + propositions: config.propositions, + authorities: config.authorities, + executors: config.executors, + ...(config.beforeCommands !== undefined ? { beforeCommands: config.beforeCommands } : {}), + }); + return { + ...host, + close(): void { + const errors: unknown[] = []; + for (const close of [() => context.close(), () => journal.close(), () => turnStore.close?.()]) { + try { + close(); + } catch (error) { + errors.push(error); + } + } + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) throw new AggregateError(errors, "failed to close continuing interaction stores"); + }, + }; + } catch (error) { + closeQuietly(() => turnStore.close?.()); + throw error; + } + } catch (error) { + closeQuietly(() => context.close()); + throw error; + } + } catch (error) { + closeQuietly(() => journal.close()); + throw error; + } +} + +function closeQuietly(close: () => void): void { + try { + close(); + } catch { + // Preserve the initialization error while releasing every opened store. + } } diff --git a/apps/inspector/src/heldout-cases.ts b/apps/inspector/src/heldout-cases.ts index d6ce74f..740c244 100644 --- a/apps/inspector/src/heldout-cases.ts +++ b/apps/inspector/src/heldout-cases.ts @@ -57,7 +57,7 @@ function referentSpec(): DecisionSpec { id: "failing", kind: "choice", decision: "referent.singular", - question: "Which task is the failing pipeline?", + question: "Which task does the user want to act on or ask about? Respect explicit exclusions and use none when no unique task is identified.", options: [{ id: "t1" }, { id: "t2" }, { id: "t3" }, { id: "none" }], noneOptions: ["none"], }; @@ -246,3 +246,38 @@ export const HELD_OUT_CASES: HeldOutCase[] = [ ], }, ]; + +/** Live language checks exclude deliberate malformed-output and synthetic-probability cases. */ +const LIVE_ACTION_CASES: HeldOutCase[] = [ + ...HELD_OUT_CASES.filter((scenario) => ["H1-pause-failing", "H4-selection-survives-restart", "H5-no-reexecution-across-restart"].includes(scenario.id)), + { + id: "L1-query-does-not-mutate", title: "a question about status does not pause a task", + entities: entities(), refs: [t1, t2, t3], descriptor, capability, + referentSpec: referentSpec(), operationSpec: operationSpec(), scripts: [], + turns: [{ kind: "language", observationId: "event:l1", utterance: "What is the current status of the nightly pipeline?", now: T1, refs: [t1, t2, t3], expect: { command: null, effectCount: 0 } }], + }, + { + id: "L2-respect-exclusion", title: "explicit exclusion leaves the nightly pipeline untouched", + entities: entities(), refs: [t1, t2, t3], descriptor, capability, + referentSpec: referentSpec(), operationSpec: operationSpec(), scripts: [], + turns: [{ kind: "language", observationId: "event:l2", utterance: "Leave the nightly pipeline alone and pause the docs refresh.", now: T1, refs: [t1, t2, t3], expect: { command: { capability: "pipeline.pause", targets: ["t2"] }, effectCount: 1 } }], + }, +]; + +export const LIVE_HELD_OUT_CASES: HeldOutCase[] = LIVE_ACTION_CASES.map((scenario) => ({ + ...scenario, + // Success counts include the explicit speech-act decision in live mode. + turns: scenario.turns.map((turn) => turn.kind === "language" && turn.expect.accepted !== undefined + ? { ...turn, expect: { ...turn.expect, accepted: turn.expect.accepted + 1 } } : turn), + actSpec: { + id: "act", kind: "choice", decision: "act", + question: "What is the user doing in this utterance? A request asks to change the world; a query asks for information and must not authorize a change.", + options: [ + { id: "request", description: "An explicit request to perform a supported operation" }, + { id: "query", description: "A question seeking information, without requesting an operation" }, + { id: "none", description: "Social speech, an unsupported act, or no clear instruction" }, + ], + noneOptions: ["none"], + }, + referentSpec: { ...scenario.referentSpec, dependsOnSpecs: ["act"] }, +})); diff --git a/apps/inspector/src/heldout-review.test.ts b/apps/inspector/src/heldout-review.test.ts new file mode 100644 index 0000000..ac4a0d6 --- /dev/null +++ b/apps/inspector/src/heldout-review.test.ts @@ -0,0 +1,73 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { mkdtempSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import * as demo from "./demo"; +import { HELD_OUT_CASES } from "./heldout-cases"; +import { createSequencedReferenceClient, runHeldOutEvaluation } from "./heldout"; +import type { JevDecisionRequest } from "@circe/perception-jev"; + +afterEach(() => vi.restoreAllMocks()); + +describe("PR9 evaluation evidence", () => { + it("fails when the intended command creates a failed effect without applying it", async () => { + const original = demo.createDemoWorld; + vi.spyOn(demo, "createDemoWorld").mockImplementation((...args) => { + const world = original(...args); + world.executors[0]!.invoke = async () => ({ ok: false, failure: { kind: "EXECUTION_ERROR" } }); + return world; + }); + const evaluation = await runHeldOutEvaluation([HELD_OUT_CASES[0]!]); + expect(evaluation.passed).toBe(false); + expect(evaluation.cases[0]!.checks).toContainEqual(expect.objectContaining({ name: "execution succeeded", ok: false })); + }); + + it("does not skip an unmatched transcript entry to find a later convenient answer", async () => { + const client = createSequencedReferenceClient([ + { decisions: ["first"], answers: [] }, + { decisions: ["later"], answers: [] }, + ]); + const request = { decisions: [{ id: "later" }] } as JevDecisionRequest; + await expect(client.complete(request)).rejects.toThrow(/no held-out scripted answer/); + }); + + it("accepts an injected client and never gives it scripts or expected answers", async () => { + const scenario = HELD_OUT_CASES[0]!; + const reference = createSequencedReferenceClient(scenario.scripts); + const requests: JevDecisionRequest[] = []; + const evaluation = await runHeldOutEvaluation([scenario], { + model: "selected-live-model", + clientForCase: () => ({ id: "injected", async complete(request) { + requests.push(request); + return reference.complete(request); + } }), + }); + expect(evaluation.passed).toBe(true); + expect(requests).toHaveLength(2); + expect(requests[0]!.model).toBe("selected-live-model"); + expect(requests[0]).not.toHaveProperty("scripts"); + expect(requests[0]).not.toHaveProperty("expect"); + expect(requests[0]!.context.utterance.text).toBe(scenario.turns[0]!.kind === "language" ? scenario.turns[0]!.utterance : ""); + }); + + it("counts actual backend calls when an observation is redelivered", async () => { + const scenario = structuredClone(HELD_OUT_CASES[0]!); + const first = scenario.turns[0]!; + if (first.kind !== "language") throw new Error("expected a language fixture"); + scenario.turns = [{ ...first, restartAfter: true }, first]; + const evaluation = await runHeldOutEvaluation([scenario]); + expect(evaluation.passed).toBe(true); + expect(evaluation.metrics.backendCalls).toBe(2); + expect(evaluation.metrics.effects).toBe(1); + }); + + it("isolates repeated evaluations under the same output directory", async () => { + const directory = mkdtempSync(join(tmpdir(), "circe-evaluation-review-")); + try { + for (let attempt = 0; attempt < 2; attempt++) { + expect((await runHeldOutEvaluation([HELD_OUT_CASES[0]!], { directory })).passed).toBe(true); + } + expect(readdirSync(directory)).toHaveLength(2); + } finally { rmSync(directory, { recursive: true, force: true }); } + }); +}); diff --git a/apps/inspector/src/heldout.ts b/apps/inspector/src/heldout.ts index 1f9d717..3a97d0c 100644 --- a/apps/inspector/src/heldout.ts +++ b/apps/inspector/src/heldout.ts @@ -1,10 +1,11 @@ -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { CapabilityDescriptor, EntityRef } from "@circe/core"; import type { CapabilitySemantics, ContextFrame } from "@circe/context"; import { ACCEPTANCE_POLICY_V1, + choiceOf, type AcceptancePolicy, type DecisionSpec, type PerceptionStatus, @@ -25,15 +26,15 @@ import { acceptedChoice, refForChoice } from "./scenario"; /** * Held-out interaction evaluation. * - * Cases are recorded conversations with a reference model: each backend call - * is answered from a transcript matched by the exact decision ids asked, and - * the harness scores what the whole product did with that answer — retrieval, + * In fixture mode each backend call is answered from a supplied script matched + * by the exact decision ids asked. The evaluation checks retrieval, * frame assembly, policy, accepted turns, compiled commands, kernel effects * and restart recovery. * * This measures the pipeline, not language understanding. The same * `JevDecisionClient` seam accepts a live transport; replacing the recorded - * client with a live one is the only change a model accuracy study needs. No + * client with a live one permits actual model runs. A representative corpus and + * outcome calibration are separate work. No * number in this harness is a claim about model quality. */ @@ -80,12 +81,13 @@ export interface HeldOutCase { refs: EntityRef[]; descriptor: CapabilityDescriptor; capability: CapabilitySemantics; + actSpec?: DecisionSpec; referentSpec: DecisionSpec; operationSpec: DecisionSpec; policy?: AcceptancePolicy; turns: HeldOutTurn[]; /** - * Recorded reference-model answers, consumed in call order. Each entry must + * Hand-written fixture answers, consumed in call order. Each entry must * match the exact decision ids of the call it answers. */ scripts: ScriptedTurn[]; @@ -130,22 +132,21 @@ export interface HeldOutEvaluation { * order. A live host would replace this with a transport-backed client. */ export function createSequencedReferenceClient(scripts: ScriptedTurn[]): JevDecisionClient { + const transcript = structuredClone(scripts); let cursor = 0; return { id: "held-out-reference", async complete(request: JevDecisionRequest): Promise { const asked = request.decisions.map((decision) => decision.id).sort(); - for (let index = cursor; index < scripts.length; index += 1) { - const script = scripts[index]!; - const wanted = [...script.decisions].sort(); - if (wanted.length === asked.length && wanted.every((id, position) => id === asked[position])) { - cursor = index + 1; - return { - status: script.status ?? "RESOLVED", - answers: structuredClone(script.answers), - latencyMs: script.latencyMs ?? 0, - }; - } + const script = transcript[cursor]; + const wanted = script ? [...script.decisions].sort() : []; + if (script && wanted.length === asked.length && wanted.every((id, position) => id === asked[position])) { + cursor += 1; + return { + status: script.status ?? "RESOLVED", + answers: structuredClone(script.answers), + latencyMs: script.latencyMs ?? 0, + }; } throw new Error( `no held-out scripted answer for decisions [${asked.join(", ")}] at call ${cursor + 1}`, @@ -162,8 +163,10 @@ export function createSequencedReferenceClient(scripts: ScriptedTurn[]): JevDeci export function selectAndActPropositions( referentSpec: DecisionSpec, operationSpec: DecisionSpec, + actSpec?: DecisionSpec, ): (run: WorkflowRun, frame: ContextFrame) => SemanticProposition[] { return (run, frame) => { + if (actSpec && acceptedChoice(run, actSpec.id) !== "request") return []; const target = acceptedChoice(run, referentSpec.id); const capability = acceptedChoice(run, operationSpec.id); if (!target || target === "none" || !capability || capability === "none") return []; @@ -179,15 +182,24 @@ export function selectAndActPropositions( refs: [ref], polarity: "include", capability, - decisions: [referentTrace.id, operationTrace.id], + decisions: [referentTrace.id, operationTrace.id, + ...(actSpec ? run.traces.filter((trace) => trace.specId === actSpec.id).map((trace) => trace.id) : [])], }, ]; }; } +export interface HeldOutEvaluationOptions { + /** Parent for preserved per-case run directories; every run gets a fresh directory. */ + directory?: string; + /** Omit for fixture playback; supply a real client for a live model evaluation. */ + clientForCase?: (scenario: HeldOutCase) => JevDecisionClient; + model?: string; +} + export async function runHeldOutEvaluation( cases: HeldOutCase[], - options: { directory?: string } = {}, + options: HeldOutEvaluationOptions = {}, ): Promise { const reports: HeldOutCaseReport[] = []; const metrics: HeldOutMetrics = { @@ -204,23 +216,30 @@ export async function runHeldOutEvaluation( for (const scenario of cases) { metrics.cases += 1; const ownedDirectory = options.directory === undefined; - const directory = - options.directory ?? mkdtempSync(join(tmpdir(), `circe-heldout-${scenario.id}-`)); + if (options.directory !== undefined) mkdirSync(options.directory, { recursive: true }); + const directory = mkdtempSync(join(options.directory ?? tmpdir(), "circe-heldout-")); const paths = { - journalPath: join(directory, `${scenario.id}.journal.jsonl`), - contextPath: join(directory, `${scenario.id}.context.jsonl`), + journalPath: join(directory, "journal.jsonl"), + contextPath: join(directory, "context.jsonl"), }; const world = createDemoWorld(scenario.entities, scenario.descriptor); - const client = createSequencedReferenceClient(scenario.scripts); - const backend = createJevBackend({ client, model: "held-out-reference" }); + const client = options.clientForCase?.(scenario) ?? createSequencedReferenceClient(scenario.scripts); + const backend = createJevBackend({ + client: { id: client.id, async complete(request) { + metrics.backendCalls += 1; + return client.complete(request); + } }, + model: options.model ?? "held-out-reference", + }); const workflow: WorkflowDefinition = { id: `${scenario.id}.v1`, stages: [ - { id: "referent", specs: () => [scenario.referentSpec] }, + ...(scenario.actSpec ? [{ id: "act", specs: () => [scenario.actSpec!] }] : []), + { id: "referent", specs: (state) => !scenario.actSpec || choiceOf(state, scenario.actSpec.id) === "request" ? [scenario.referentSpec] : [] }, { id: "operation", specs: () => [scenario.operationSpec] }, ], }; - const propositions = selectAndActPropositions(scenario.referentSpec, scenario.operationSpec); + const propositions = selectAndActPropositions(scenario.referentSpec, scenario.operationSpec, scenario.actSpec); const config = { interactionId: `interaction:${scenario.id}`, capabilities: [scenario.capability], @@ -232,11 +251,12 @@ export async function runHeldOutEvaluation( executors: world.executors, }; - let host = createFileContinuingInteractionHost({ ...config, ...paths }); + let host: ReturnType | undefined; const checks: HeldOutCheck[] = []; const commands: HeldOutCaseReport["commands"] = []; const effectIds: string[] = []; let turnIndex = 0; + const delivered = new Set(); const check = (name: string, ok: boolean, detail?: string): void => { checks.push({ turn: turnIndex, name, ok, ...(detail !== undefined ? { detail } : {}) }); @@ -246,6 +266,7 @@ export async function runHeldOutEvaluation( }; try { + host = createFileContinuingInteractionHost({ ...config, ...paths }); for (const turn of scenario.turns) { turnIndex += 1; metrics.turns += 1; @@ -260,13 +281,13 @@ export async function runHeldOutEvaluation( expectEqual("ui scope", result.scope.map((ref) => ref.id), turn.expect.scope); } } else { + const invokedBefore = world.invoked.length; const report = await host.runTurn({ observationId: turn.observationId, utterance: turn.utterance, now: turn.now, ...(turn.refs !== undefined ? { refs: turn.refs } : {}), }); - metrics.backendCalls += report.run.calls; metrics.acceptedDecisions += report.run.state.accepted.size; const turnCommands = report.compiled.commands.map((command) => ({ capability: command.capability, @@ -288,8 +309,15 @@ export async function runHeldOutEvaluation( } if (expect.command === null) { expectEqual("no compiled command", turnCommands.length, 0); + expectEqual("no applied mutation", world.invoked.length, invokedBefore); } else if (expect.command !== undefined) { expectEqual("compiled command", turnCommands, [expect.command]); + check("execution succeeded", report.results.length === turnCommands.length && + report.results.every((result) => !result.errors?.length && result.effects.length > 0 && + result.effects.every((effect) => effect.status === "SUCCEEDED")), + "Every expected command must produce a successful effect, not merely an effect id."); + expectEqual("applied targets", world.invoked.slice(invokedBefore).map((ref) => ref.id), + delivered.has(turn.observationId) ? [] : expect.command.targets); } if (expect.effectCount !== undefined) { expectEqual("cumulative effects", effectIds.length, expect.effectCount); @@ -308,6 +336,7 @@ export async function runHeldOutEvaluation( `expected at least ${expect.recentReferentsAtLeast}, got ${report.frame.interaction.recentReferents.length}`, ); } + delivered.add(turn.observationId); if (turnCommands.length === 0) { // A run that failed validation is not an abstention; it produced // no binding because the answer was unusable, not because the @@ -321,9 +350,12 @@ export async function runHeldOutEvaluation( host = createFileContinuingInteractionHost({ ...config, ...paths }); } } + } catch (error) { + check("case completed", false, error instanceof Error ? error.message : "Case threw an unknown error"); } finally { - host.close(); - if (ownedDirectory) rmSync(directory, { recursive: true, force: true }); + try { host?.close(); } + catch (error) { check("stores closed", false, error instanceof Error ? error.message : "Close failed"); } + finally { if (ownedDirectory) rmSync(directory, { recursive: true, force: true }); } } const failed = checks.filter((entry) => !entry.ok); diff --git a/apps/inspector/src/host.test.ts b/apps/inspector/src/host.test.ts index 0c1bdd2..7b2b5e1 100644 --- a/apps/inspector/src/host.test.ts +++ b/apps/inspector/src/host.test.ts @@ -68,6 +68,7 @@ describe("continuing interaction host", () => { now: T1, refs: [refA, refB], }); + expect(host.context.decisions().map((trace) => trace.id)).toEqual(first.run.traces.map((trace) => trace.id)); expect(first.frame.interaction.activeScope?.refs.map((ref) => ref.id)).toEqual(["A", "B"]); expect(first.frame.interaction.recentReferents.length).toBe(1); @@ -227,6 +228,7 @@ describe("continuing interaction host", () => { utterance: "Leave the sketchy auth one alone and continue the other one.", now: T1, }); + expect(first.context.decisions().map((trace) => trace.id)).toEqual(report.run.traces.map((trace) => trace.id)); const effectIds = report.results.flatMap((result) => result.effects).map((effect) => effect.id); expect(effectIds).toHaveLength(1); first.close(); @@ -244,6 +246,256 @@ describe("continuing interaction host", () => { rmSync(directory, { recursive: true, force: true }); } }); + + it("replays the pinned interpretation after restart without reasking or redispatching", async () => { + const directory = mkdtempSync(join(tmpdir(), "circe-host-replay-")); + const paths = { journalPath: join(directory, "journal.jsonl"), contextPath: join(directory, "context.jsonl") }; + const input = { + observationId: "event:retry", + utterance: "Leave the sketchy auth one alone and continue the other one.", + now: T1, + refs: [refA, refB], + }; + try { + const firstWorld = createDemoWorld(demoEntities()); + const first = createFileContinuingInteractionHost({ + ...hostConfig(firstWorld, { sketchy: "B", operation: "task.resume" }), + ...paths, + }); + const original = await first.runTurn(input); + expect(firstWorld.invoked).toHaveLength(1); + first.close(); + + const secondWorld = createDemoWorld(demoEntities()); + let decisions = 0; + const backend = createScriptedBackend({ sketchy: "B", operation: "task.resume" }); + const countingBackend = { + ...backend, + async decide(...args: Parameters) { + decisions += 1; + return backend.decide(...args); + }, + }; + const second = createFileContinuingInteractionHost({ + ...hostConfig(secondWorld, { sketchy: "B", operation: "task.resume" }), + backend: countingBackend, + ...paths, + }); + const replay = await second.runTurn(input); + expect(decisions).toBe(0); + expect(replay.run.runId).toBe(original.run.runId); + expect(replay.compiled.commands).toEqual(original.compiled.commands); + expect(secondWorld.invoked).toHaveLength(0); + expect(replay.results).toEqual(original.results); + await expect(second.runTurn({ ...input, utterance: "A different utterance." })).rejects.toThrow( + "observation identity event:retry was reused with different turn input", + ); + expect(decisions).toBe(0); + second.close(); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it("reconciles a lost invocation response after restart without invoking again", async () => { + const directory = mkdtempSync(join(tmpdir(), "circe-host-unknown-")); + const paths = { journalPath: join(directory, "journal.jsonl"), contextPath: join(directory, "context.jsonl") }; + const world = createDemoWorld(demoEntities()); + const executor = world.executors[0]!; + const invoke = executor.invoke.bind(executor); + executor.invoke = async (request) => { await invoke(request); throw new Error("lost response"); }; + try { + const config = { ...hostConfig(world, { sketchy: "B", operation: "task.resume" }), ...paths }; + const first = createFileContinuingInteractionHost(config); + const report = await first.runTurn({ observationId: "event:unknown", utterance: "Continue the other task.", now: T1, refs: [refA, refB] }); + const effect = [...report.state.effects.values()][0]!; + expect(effect.status).toBe("UNKNOWN"); + expect(world.invoked).toHaveLength(1); + first.close(); + const second = createFileContinuingInteractionHost(config); + await second.reconcile(effect.id, T2); + expect(second.state().effects.get(effect.id)!.status).toBe("SUCCEEDED"); + expect(world.invoked).toHaveLength(1); + second.close(); + } finally { rmSync(directory, { recursive: true, force: true }); } + }); + + it("allows typed confirmation to continue a gated turn after restart", async () => { + const directory = mkdtempSync(join(tmpdir(), "circe-host-confirm-")); + const paths = { journalPath: join(directory, "journal.jsonl"), contextPath: join(directory, "context.jsonl") }; + const gatedCapability = { ...taskResume, policy: { ...taskResume.policy, commit: "affirmative-confirmation" as const } }; + try { + const firstWorld = createDemoWorld(demoEntities(), gatedCapability); + const first = createFileContinuingInteractionHost({ + ...hostConfig(firstWorld, { sketchy: "B", operation: "task.resume" }), + ...paths, + }); + const proposed = await first.runTurn({ + observationId: "event:gated", + utterance: "Leave the sketchy auth one alone and continue the other one.", + now: T1, + refs: [refA, refB], + }); + expect(proposed.results.flatMap((result) => result.effects)).toHaveLength(0); + expect(first.state().presentedGateId).toBeTruthy(); + first.close(); + + const secondWorld = createDemoWorld(demoEntities(), gatedCapability); + const second = createFileContinuingInteractionHost({ + ...hostConfig(secondWorld, { sketchy: "B", operation: "task.resume" }), + ...paths, + }); + const transactionId = second.state().presentedGateId!; + const intent = second.state().intents.get(second.state().transactions.get(transactionId)!.intentId)!; + const confirmed = await second.command({ + kind: "confirm", + commandId: "event:gated-confirm", + transactionId, + intentRevision: intent.revision, + }, T2); + expect(confirmed.effects.some((effect) => effect.status === "SUCCEEDED")).toBe(true); + expect(secondWorld.invoked.map((ref) => ref.id)).toEqual(["A"]); + second.close(); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it("does not reinterpret an abstained observation when the same identity is retried", async () => { + const directory = mkdtempSync(join(tmpdir(), "circe-host-abstain-")); + const paths = { journalPath: join(directory, "journal.jsonl"), contextPath: join(directory, "context.jsonl") }; + const input = { observationId: "event:abstain", utterance: "Maybe continue something.", now: T1, refs: [refA, refB] }; + try { + const firstBackend = createScriptedBackend({ sketchy: "B", operation: "task.resume" }); + const ambiguousBackend = { + ...firstBackend, + async decide(...args: Parameters) { + const result = await firstBackend.decide(...args); + return { + ...result, + answers: result.answers.map((answer) => answer.kind === "choice" + ? { ...answer, confidence: 0.4, probabilities: Object.fromEntries(Object.keys(answer.probabilities).map((id) => [id, 1 / Object.keys(answer.probabilities).length])) } + : answer), + }; + }, + }; + const world = createDemoWorld(demoEntities()); + const first = createFileContinuingInteractionHost({ + ...hostConfig(world, { sketchy: "B", operation: "task.resume" }), + backend: ambiguousBackend, + ...paths, + }); + const original = await first.runTurn(input); + expect(original.compiled.commands).toHaveLength(0); + expect(world.invoked).toHaveLength(0); + first.close(); + + let decisions = 0; + const retryBackend = { + ...firstBackend, + async decide(...args: Parameters) { + decisions += 1; + return firstBackend.decide(...args); + }, + }; + const retry = createFileContinuingInteractionHost({ + ...hostConfig(createDemoWorld(demoEntities()), { sketchy: "B", operation: "task.resume" }), + backend: retryBackend, + ...paths, + }); + const replay = await retry.runTurn(input); + expect(decisions).toBe(0); + expect(replay.compiled.commands).toEqual([]); + expect(replay.run.runId).toBe(original.run.runId); + retry.close(); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it("recovers a pinned turn when failure happens immediately before dispatch", async () => { + const directory = mkdtempSync(join(tmpdir(), "circe-host-cut-")); + const paths = { journalPath: join(directory, "journal.jsonl"), contextPath: join(directory, "context.jsonl") }; + const input = { observationId: "event:dispatch-cut", utterance: "Continue the other task.", now: T1, refs: [refA, refB] }; + try { + const firstWorld = createDemoWorld(demoEntities()); + const first = createFileContinuingInteractionHost({ + ...hostConfig(firstWorld, { sketchy: "B", operation: "task.resume" }), + ...paths, + beforeCommands: () => { throw new Error("simulated interruption before dispatch"); }, + }); + await expect(first.runTurn(input)).rejects.toThrow("simulated interruption before dispatch"); + expect(firstWorld.invoked).toHaveLength(0); + first.close(); + + const secondWorld = createDemoWorld(demoEntities()); + let decisions = 0; + const backend = createScriptedBackend({ sketchy: "B", operation: "task.resume" }); + const retryBackend = { + ...backend, + async decide(...args: Parameters) { + decisions += 1; + return backend.decide(...args); + }, + }; + const second = createFileContinuingInteractionHost({ + ...hostConfig(secondWorld, { sketchy: "B", operation: "task.resume" }), + backend: retryBackend, + ...paths, + }); + const recovered = await second.runTurn(input); + expect(decisions).toBe(0); + expect(recovered.compiled.commands).toHaveLength(1); + expect(secondWorld.invoked.map((ref) => ref.id)).toEqual(["A"]); + second.close(); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it("serializes simultaneous turns and persists their decision provenance before execution", async () => { + const world = createDemoWorld(demoEntities()); + const baseBackend = createScriptedBackend({ sketchy: "B", operation: "task.resume" }); + let active = 0; + let maxActive = 0; + const backend = { + ...baseBackend, + async decide(frame: Parameters[0], specs: Parameters[1]) { + active += 1; + maxActive = Math.max(maxActive, active); + await new Promise((resolve) => setTimeout(resolve, 5)); + try { + return await baseBackend.decide(frame, specs); + } finally { + active -= 1; + } + }, + }; + const host = createContinuingInteractionHost({ + ...hostConfig(world, { sketchy: "B", operation: "task.resume" }), + backend, + context: createInMemoryContextStore(), + journal: createInMemoryJournal(), + }); + + const queuedInput = { + observationId: "event:parallel-2", + utterance: "Continue the task again.", + now: T2, + refs: [refA, refB], + }; + const firstPromise = host.runTurn({ observationId: "event:parallel-1", utterance: "Continue the task.", now: T1, refs: [refA, refB] }); + const secondPromise = host.runTurn(queuedInput); + queuedInput.utterance = "mutated after enqueue"; + const [first, second] = await Promise.all([firstPromise, secondPromise]); + + expect(maxActive).toBe(1); + expect(second.frame.utterance.text).toBe("Continue the task again."); + expect(host.context.decisions().map((trace) => trace.id)).toEqual([ + ...first.run.traces.map((trace) => trace.id), + ...second.run.traces.map((trace) => trace.id), + ]); + }); }); function effectsOf(report: { results: Array<{ effects: Array<{ id: string; status: string }> }> }) { diff --git a/apps/inspector/src/host.ts b/apps/inspector/src/host.ts index fa2e83f..65136a8 100644 --- a/apps/inspector/src/host.ts +++ b/apps/inspector/src/host.ts @@ -5,9 +5,11 @@ import { type CapabilityDescriptor, type CapabilityExecutor, type EntityRef, + type EffectId, type Intent, type InteractionJournal, type InteractionState, + type KernelCommand, type Observation, type Referent, type Transaction, @@ -18,6 +20,7 @@ import { ingestWorldView, type CapabilitySemantics, type ContextFrame, + type DecisionTrace, type FrameInteraction, type FrameLiveObligation, type FramePendingIntent, @@ -35,6 +38,11 @@ import { type WorkflowDefinition, type WorkflowRun, } from "@circe/perception"; +import { + createInMemoryAcceptedTurnStore, + type AcceptedTurnStore, + type AcceptedTurnRecord, +} from "./turn-store"; /** * Continuing interaction host. @@ -70,6 +78,8 @@ export interface ContinuingInteractionConfig { /** durable or in-memory; the host never closes what it does not own */ context: SemanticContextStoreContract; journal: InteractionJournal; + /** optional durable pin for the first interpretation of each observation */ + turnStore?: AcceptedTurnStore; capabilities: CapabilitySemantics[]; workflow: WorkflowDefinition; backend: PerceptionBackend; @@ -105,6 +115,10 @@ export interface ContinuingTurnReport { export interface ContinuingInteractionHost { readonly context: SemanticContextStoreContract; runTurn(input: ContinuingTurnInput): Promise; + /** Drive a typed kernel command (confirm/cancel/resume/revise/propose). */ + command(command: KernelCommand, now: string): Promise; + /** Resolve an UNKNOWN dispatch from its recorded receipt. */ + reconcile(effectId: EffectId, now: string): Promise; /** * A non-language kernel input turn (UI focus/selection, world events). * Language never goes through here; the perception path is `runTurn`. @@ -215,6 +229,15 @@ export function createContinuingInteractionHost( config: ContinuingInteractionConfig, ): ContinuingInteractionHost { let clockNow: string | undefined; + const turnStore = config.turnStore ?? createInMemoryAcceptedTurnStore(); + // A continuing interaction has one discourse timeline. Serialize ingestion + // and language turns so each frame is grounded in the previous turn's state. + let turnQueue: Promise = Promise.resolve(); + function enqueue(work: () => Promise): Promise { + const result = turnQueue.then(work); + turnQueue = result.then(() => undefined, () => undefined); + return result; + } const runtime = createRuntime({ ports: { authorityFor: (ref) => { @@ -256,51 +279,117 @@ export function createContinuingInteractionHost( context: config.context, state: () => runtime.state(), - async ingest(observation: Observation): Promise { - clockNow = observation.at; - return runtime.ingest(observation); + ingest(observation: Observation): Promise { + const stableObservation = structuredClone(observation); + return enqueue(async () => { + clockNow = stableObservation.at; + return runtime.ingest(stableObservation); + }); }, - async runTurn(input: ContinuingTurnInput): Promise { - clockNow = input.now; - const state = runtime.state(); - const scopeId = state.scopeStack[state.scopeStack.length - 1]; - const scopeRefs = scopeId ? (state.scopes.get(scopeId)?.refs ?? []) : []; - const refs = input.refs ?? scopeRefs; - await observe(refs); - - const { frame } = await assembleFrame(config.context, { - interactionId: config.interactionId, - observationId: input.observationId, - utterance: { text: input.utterance, source: "text" }, - interaction: frameInteractionFor(runtime.state(), refs, descriptors()), - capabilities: config.capabilities, - now: input.now, + command(command: KernelCommand, now: string): Promise { + const stableCommand = structuredClone(command); + return enqueue(async () => { + clockNow = now; + return runtime.command(stableCommand); }); + }, - const run = await runPerceptionWorkflow({ - backend: config.backend, - frame, - workflow: config.workflow, - policy: config.policy, - now: input.now, - runId: `host:${config.interactionId}:${input.observationId}`, + reconcile(effectId: EffectId, now: string): Promise { + return enqueue(async () => { + clockNow = now; + return runtime.reconcile(effectId); }); + }, - const turn = semanticTurnFromRun(run, config.propositions(run, frame)); - const compiled = compileSemanticTurn(turn, { - frame, - commandIdFor: (_proposition, index) => `${config.interactionId}:${input.observationId}:${index}`, - }); + runTurn(input: ContinuingTurnInput): Promise { + const stableInput = structuredClone(input); + return enqueue(async () => { + const input = stableInput; + const key = `${config.interactionId}:${input.observationId}`; + const inputFingerprint = JSON.stringify({ + observationId: input.observationId, + utterance: input.utterance, + now: input.now, + refs: input.refs === undefined + ? "" + : input.refs.map((ref) => ({ + namespace: ref.namespace, + type: ref.type, + id: ref.id, + authority: ref.authority, + })), + }); + const pinned = turnStore.get(key); + if (pinned && pinned.inputFingerprint !== inputFingerprint) { + throw new Error(`observation identity ${input.observationId} was reused with different turn input`); + } - config.beforeCommands?.(frame); + clockNow = input.now; + let frame: ContextFrame; + let run: WorkflowRun; + let turn: SemanticTurn; + let compiled: CompiledTurn; + if (pinned) { + ({ frame, run, turn, compiled } = pinned); + } else { + const state = runtime.state(); + const scopeId = state.scopeStack[state.scopeStack.length - 1]; + const scopeRefs = scopeId ? (state.scopes.get(scopeId)?.refs ?? []) : []; + const refs = input.refs ?? scopeRefs; + await observe(refs); - const results: TurnResult[] = []; - for (const command of compiled.commands) { - results.push(await runtime.command(command)); - } + ({ frame } = await assembleFrame(config.context, { + interactionId: config.interactionId, + observationId: input.observationId, + utterance: { text: input.utterance, source: "text" }, + interaction: frameInteractionFor(runtime.state(), refs, descriptors()), + capabilities: config.capabilities, + now: input.now, + })); + + run = await runPerceptionWorkflow({ + backend: config.backend, + frame, + workflow: config.workflow, + policy: config.policy, + now: input.now, + runId: `host:${config.interactionId}:${frame.id}`, + }); + + turn = semanticTurnFromRun(run, config.propositions(run, frame)); + compiled = compileSemanticTurn(turn, { + frame, + commandIdFor: (_proposition, index) => `${config.interactionId}:${input.observationId}:${index}`, + }); + const record: AcceptedTurnRecord = { key, inputFingerprint, frame, run, turn, compiled }; + // Pin the first interpretation before trace writes or any command dispatch. + turnStore.record(record); + } - return { frame, run, turn, compiled, results, state: runtime.state() }; + persistDecisionTraces(config.context, run.traces); + config.beforeCommands?.(frame); + + const results: TurnResult[] = []; + for (const command of compiled.commands) { + results.push(await runtime.command(command)); + } + + return { frame, run, turn, compiled, results, state: runtime.state() }; + }); }, }; } + +function persistDecisionTraces(store: SemanticContextStoreContract, traces: DecisionTrace[]): void { + for (const trace of traces) { + const existing = store.decision(trace.id); + if (existing) { + if (JSON.stringify(existing) !== JSON.stringify(trace)) { + throw new Error(`decision identity ${trace.id} was reused with different provenance`); + } + continue; + } + store.recordDecision(trace); + } +} diff --git a/apps/inspector/src/live-cli.ts b/apps/inspector/src/live-cli.ts new file mode 100644 index 0000000..6dd733a --- /dev/null +++ b/apps/inspector/src/live-cli.ts @@ -0,0 +1,26 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { createTypeSafeJevClient } from "@circe/perception-jev"; +import { LIVE_HELD_OUT_CASES } from "./heldout-cases"; +import { runHeldOutEvaluation } from "./heldout"; + +// This command sends synthetic corpus data to the selected TypeSafe model. +// Executors are the same local test doubles used by the contract suite. +const model = process.env.TYPESAFE_MODEL; +if (!process.env.TYPESAFE_API_KEY || !model) { + console.error("Live evaluation needs TYPESAFE_API_KEY and TYPESAFE_MODEL. No model call was made."); + process.exitCode = 2; +} else { + const client = createTypeSafeJevClient(); + const directory = process.env.CIRCE_EVAL_DIRECTORY; + const evaluation = await runHeldOutEvaluation(LIVE_HELD_OUT_CASES, { + model, clientForCase: () => client, ...(directory ? { directory } : {}), + }); + const report = { mode: "live", model, at: new Date().toISOString(), ...evaluation }; + if (directory) { + mkdirSync(directory, { recursive: true }); + writeFileSync(join(directory, "live-evaluation.json"), JSON.stringify(report, null, 2) + "\n"); + } + console.log(JSON.stringify(report, null, 2)); + if (!evaluation.passed) process.exitCode = 1; +} diff --git a/apps/inspector/src/live-evaluation.test.ts b/apps/inspector/src/live-evaluation.test.ts new file mode 100644 index 0000000..bac50a2 --- /dev/null +++ b/apps/inspector/src/live-evaluation.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { LIVE_HELD_OUT_CASES } from "./heldout-cases"; +import { createSequencedReferenceClient, runHeldOutEvaluation } from "./heldout"; + +describe("live evaluation workflow contracts (mocked client, not accuracy results)", () => { + it("does not ask for a capability after an accepted information query", async () => { + const scenario = LIVE_HELD_OUT_CASES.find((entry) => entry.id === "L1-query-does-not-mutate")!; + const client = createSequencedReferenceClient([{ decisions: ["act"], answers: [{ + id: "act", kind: "choice", choice: "query", probabilities: { request: 0, query: 1, none: 0 }, confidence: 1, + }] }]); + const evaluation = await runHeldOutEvaluation([scenario], { model: "test", clientForCase: () => client }); + expect(evaluation.passed).toBe(true); + expect(evaluation.metrics).toMatchObject({ backendCalls: 1, commands: 0, effects: 0 }); + }); + + it("carries accepted act, target and capability evidence through a restart", async () => { + const scenario = LIVE_HELD_OUT_CASES.find((entry) => entry.id === "H4-selection-survives-restart")!; + const client = createSequencedReferenceClient([ + { decisions: ["act"], answers: [{ id: "act", kind: "choice", choice: "request", probabilities: { request: 1, query: 0, none: 0 }, confidence: 1 }] }, + ...scenario.scripts, + ]); + const evaluation = await runHeldOutEvaluation([scenario], { model: "test", clientForCase: () => client }); + expect(evaluation.passed).toBe(true); + expect(evaluation.metrics).toMatchObject({ backendCalls: 3, acceptedDecisions: 3, commands: 1, effects: 1 }); + }); +}); diff --git a/apps/inspector/src/turn-store.test.ts b/apps/inspector/src/turn-store.test.ts new file mode 100644 index 0000000..0ccc644 --- /dev/null +++ b/apps/inspector/src/turn-store.test.ts @@ -0,0 +1,96 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { createInMemoryJournal } from "@circe/core"; +import { createInMemoryContextStore } from "@circe/context"; +import { ACCEPTANCE_POLICY_V1 } from "@circe/perception"; +import { createDemoWorld, taskResume } from "./demo"; +import { createContinuingInteractionHost } from "./host"; +import { createScriptedBackend, demoEntities, difficultUtteranceWorkflow, propositionsFromDecisions, refA, refB } from "./scenario"; +import { createFileAcceptedTurnStore, createInMemoryAcceptedTurnStore, type AcceptedTurnRecord } from "./turn-store"; + +async function recordedTurn(): Promise { + let captured: AcceptedTurnRecord | undefined; + const world = createDemoWorld(demoEntities()); + const backing = createInMemoryAcceptedTurnStore(); + const host = createContinuingInteractionHost({ + interactionId: "interaction:inspector", + context: createInMemoryContextStore(), journal: createInMemoryJournal(), + turnStore: { get: backing.get, record(record) { backing.record(record); captured = structuredClone(record); } }, + capabilities: [{ capabilityId: taskResume.id, description: "Continue a task", semanticEffects: "The task becomes active", slots: [{ name: "targets", description: "Tasks to continue" }] }], + workflow: difficultUtteranceWorkflow, + backend: createScriptedBackend({ sketchy: "B", operation: "task.resume" }), + policy: ACCEPTANCE_POLICY_V1, propositions: propositionsFromDecisions, + authorities: world.authorities, executors: world.executors, + }); + await host.runTurn({ observationId: "event:record", utterance: "Leave the sketchy auth one alone and continue the other one.", now: "2026-09-24T09:00:00.000Z", refs: [refA, refB] }); + if (!captured) throw new Error("host did not record a turn"); + return captured; +} + +async function withLog(run: (path: string, record: AcceptedTurnRecord) => void) { + const directory = mkdtempSync(join(tmpdir(), "circe-turn-store-")); + try { run(join(directory, "turns.jsonl"), await recordedTurn()); } + finally { rmSync(directory, { recursive: true, force: true }); } +} + +describe("accepted turn durability boundary", () => { + it("round trips workflow maps and returns detached snapshots", async () => { + await withLog((path, record) => { + const store = createFileAcceptedTurnStore(path); + const original = structuredClone(record); + store.record(record); + record.run.state.accepted.clear(); + expect(store.get(original.key)).toEqual(original); + store.get(original.key)!.run.state.answers.clear(); + store.close?.(); + const reopened = createFileAcceptedTurnStore(path); + expect(reopened.get(original.key)).toEqual(original); + expect(reopened.get(original.key)!.run.state.accepted.size).toBeGreaterThan(0); + reopened.close?.(); + }); + }); + + it("detects conflicts in Map contents rather than comparing empty JSON objects", async () => { + await withLog((path, record) => { + for (const store of [createFileAcceptedTurnStore(path), createInMemoryAcceptedTurnStore()]) { + store.record(record); + const changed = structuredClone(record); + changed.run.state.accepted.clear(); + expect(() => store.record(changed)).toThrow("different content"); + store.close?.(); + } + }); + }); + + it("refuses records with changed provenance or compiled commands", async () => { + await withLog((path, record) => { + const store = createFileAcceptedTurnStore(path); + store.record(record); store.close?.(); + const serialized = readFileSync(path, "utf8"); + const wrongFrame = JSON.parse(serialized); + wrongFrame.turn.frameHash = "different"; + writeFileSync(path, JSON.stringify(wrongFrame) + "\n"); + expect(() => createFileAcceptedTurnStore(path)).toThrow("provenance"); + const wrongCommand = JSON.parse(serialized); + wrongCommand.compiled.commands[0].capability = "different.operation"; + writeFileSync(path, JSON.stringify(wrongCommand) + "\n"); + expect(() => createFileAcceptedTurnStore(path)).toThrow("commands do not match"); + }); + }); + + it("refuses duplicate and malformed serialized Map entries", async () => { + await withLog((path, record) => { + const store = createFileAcceptedTurnStore(path); + store.record(record); store.close?.(); + const serialized = readFileSync(path, "utf8"); + for (const extra of [JSON.parse(serialized).run.state.answers[0], ["incomplete"]]) { + const malformed = JSON.parse(serialized); + malformed.run.state.answers.push(extra); + writeFileSync(path, JSON.stringify(malformed) + "\n"); + expect(() => createFileAcceptedTurnStore(path)).toThrow("workflow entry"); + } + }); + }); +}); diff --git a/apps/inspector/src/turn-store.ts b/apps/inspector/src/turn-store.ts new file mode 100644 index 0000000..0c7652f --- /dev/null +++ b/apps/inspector/src/turn-store.ts @@ -0,0 +1,169 @@ +import type { CompiledTurn, SemanticTurn, WorkflowRun } from "@circe/perception"; +import type { ContextFrame } from "@circe/context"; +import { compileSemanticTurn } from "@circe/perception"; +import { createAppendOnlyLog } from "@circe/file-log"; + +export interface AcceptedTurnRecord { + key: string; + inputFingerprint: string; + frame: ContextFrame; + run: WorkflowRun; + turn: SemanticTurn; + compiled: CompiledTurn; +} + +export interface AcceptedTurnStore { + get(key: string): AcceptedTurnRecord | undefined; + record(record: AcceptedTurnRecord): void; + close?(): void; +} + +export function createInMemoryAcceptedTurnStore(): AcceptedTurnStore { + const records = new Map(); + return { + get: (key) => { + const record = records.get(key); + return record ? structuredClone(record) : undefined; + }, + record(record) { + const existing = records.get(record.key); + const normalized = validateAcceptedTurnRecord(record); + if (existing && encodeAcceptedTurnRecord(existing) !== encodeAcceptedTurnRecord(normalized)) { + throw new Error(`accepted turn ${record.key} was recorded with different content`); + } + if (!existing) records.set(record.key, structuredClone(normalized)); + }, + }; +} + +export function createFileAcceptedTurnStore(path: string, options: { durable?: boolean } = {}): AcceptedTurnStore { + const log = createAppendOnlyLog(path, { + label: "accepted turn", + ...(options.durable !== undefined ? { durable: options.durable } : {}), + decode: decodeAcceptedTurnRecord, + encode: encodeAcceptedTurnRecord, + }); + const records = new Map(); + for (const record of log.read()) { + const existing = records.get(record.key); + if (existing && encodeAcceptedTurnRecord(existing) !== encodeAcceptedTurnRecord(record)) { + log.close(); + throw new Error(`conflicting accepted turn records for ${record.key}`); + } + records.set(record.key, record); + } + return { + get: (key) => { + const record = records.get(key); + return record ? structuredClone(record) : undefined; + }, + record(record) { + const existing = records.get(record.key); + const normalized = decodeAcceptedTurnRecord(JSON.parse(encodeAcceptedTurnRecord(record))); + if (existing) { + if (encodeAcceptedTurnRecord(existing) !== encodeAcceptedTurnRecord(normalized)) { + throw new Error(`accepted turn ${record.key} was recorded with different content`); + } + return; + } + log.append(normalized); + records.set(record.key, structuredClone(normalized)); + }, + close: () => log.close(), + }; +} + +export function validateAcceptedTurnRecord(value: unknown): AcceptedTurnRecord { + if (!value || typeof value !== "object") throw new Error("accepted turn record must be an object"); + const record = value as Partial; + if ( + typeof record.key !== "string" || + typeof record.inputFingerprint !== "string" || + !record.frame || typeof record.frame !== "object" || + !record.run || typeof record.run !== "object" || + !record.turn || typeof record.turn !== "object" || + !record.compiled || typeof record.compiled !== "object" || + !Array.isArray(record.compiled.commands) + ) throw new Error("invalid accepted turn record"); + const run = record.run as WorkflowRun; + if (!run.state || !(run.state.answers instanceof Map) || !(run.state.accepted instanceof Map) || + !(run.state.outcomes instanceof Map) || !Array.isArray(run.state.traces) || !Array.isArray(run.state.notes) || + !Array.isArray(run.traces) || !Array.isArray(run.failures) || + typeof run.runId !== "string" || typeof run.frameId !== "string" || typeof run.frameHash !== "string") { + throw new Error("invalid accepted turn workflow snapshot"); + } + const frame = record.frame as ContextFrame; + const turn = record.turn as SemanticTurn; + if ( + typeof frame.id !== "string" || typeof frame.hash !== "string" || + typeof frame.interactionId !== "string" || typeof frame.observationId !== "string" || + typeof frame.utterance?.text !== "string" || turn.frameId !== frame.id || + turn.frameHash !== frame.hash || run.frameId !== frame.id || run.frameHash !== frame.hash || + run.runId !== turn.runId || turn.traces === undefined || + encodeValue(run.traces) !== encodeValue(turn.traces) || + encodeValue(run.traces) !== encodeValue(run.state.traces) || + record.key !== `${frame.interactionId}:${frame.observationId}` + ) throw new Error("accepted turn provenance does not match its frame"); + const expected = compileSemanticTurn(turn, { + frame, + commandIdFor: (_proposition, index) => `${frame.interactionId}:${frame.observationId}:${index}`, + }); + if (encodeValue(expected) !== encodeValue(record.compiled)) { + throw new Error("accepted turn commands do not match the compiled semantic turn"); + } + return record as AcceptedTurnRecord; +} + +function encodeValue(value: unknown): string { + return JSON.stringify(value); +} + +function encodeAcceptedTurnRecord(record: AcceptedTurnRecord): string { + return JSON.stringify({ + ...record, + run: { + ...record.run, + state: { + answers: [...record.run.state.answers], + accepted: [...record.run.state.accepted], + outcomes: [...record.run.state.outcomes], + traces: record.run.state.traces, + notes: record.run.state.notes, + }, + }, + }); +} + +function decodeAcceptedTurnRecord(value: unknown): AcceptedTurnRecord { + if (!value || typeof value !== "object") throw new Error("accepted turn record must be an object"); + const record = value as Record; + const run = record.run as Record | undefined; + const state = run?.state as Record | undefined; + if (!state || !Array.isArray(state.answers) || !Array.isArray(state.accepted) || !Array.isArray(state.outcomes)) { + throw new Error("invalid accepted turn workflow state"); + } + return validateAcceptedTurnRecord({ + ...record, + run: { + ...run, + state: { + answers: decodeEntries(state.answers), + accepted: decodeEntries(state.accepted), + outcomes: decodeEntries(state.outcomes), + traces: state.traces, + notes: state.notes, + }, + }, + }); +} + +function decodeEntries(entries: unknown[]): Map { + const result = new Map(); + for (const entry of entries) { + if (!Array.isArray(entry) || entry.length !== 2 || typeof entry[0] !== "string" || result.has(entry[0])) { + throw new Error("invalid or duplicate accepted turn workflow entry"); + } + result.set(entry[0], entry[1]); + } + return result; +} diff --git a/docs/SEMANTIC-CONTEXT-MODEL.md b/docs/SEMANTIC-CONTEXT-MODEL.md index 5d02ba7..9234474 100644 --- a/docs/SEMANTIC-CONTEXT-MODEL.md +++ b/docs/SEMANTIC-CONTEXT-MODEL.md @@ -6,8 +6,8 @@ candidate-source (seed) layer, bounded ContextFrame assembly with a **materializ semantic neighbourhood, retrieval metrics, transitive dependency invalidation, and authority-safe identity exist and are exercised by the eval suites. Perception contracts, the staged workflow orchestrator, and the acceptance policy live in `@circe/perception`. -The injected Jev client adapter is implemented in `@circe/perception-jev`; a live SDK -integration is still supplied by the host. +The injected Jev adapter and native-fetch TypeSafe transport are implemented in +`@circe/perception-jev`. Live accuracy evaluation still requires configured API access. Corrective pass (review of `e40b533`), all in this revision: @@ -48,11 +48,14 @@ deterministic replay on open. Kernel state has separate journal-backed recovery `@circe/journal-file`. One process owns a store path at a time; multi-process coordination and compaction are deferred. -The two logs are separate durability boundaries. The kernel journal is authoritative for -commands, confirmations and effects; the context log holds evidence. A crash between the -two can leave evidence behind or missing, and both are safe: a host re-grounds evidence by -observing authorities at the next turn, and an unrecorded command is replayed from the -journal, not the context log. +The kernel journal is authoritative for commands, confirmations and effects. The context +log holds evidence. The continuing host also records the first accepted-turn snapshot in +a separate log before persisting its decision traces and dispatching commands. That record +includes abstentions; a retry cannot reinterpret the same observation into new work. +If a crash separates these writes, replay restores missing decision traces from the saved +turn and redelivers the same command IDs through the kernel's recovery checks. Saved turns +do not replace kernel authorization or executor reconciliation. These files are separate +append boundaries, not an atomic multi-file transaction. | Durable in Circe | Not state | |---|---| diff --git a/packages/context-file/src/codec.ts b/packages/context-file/src/codec.ts index 8d72ce0..2022f2b 100644 --- a/packages/context-file/src/codec.ts +++ b/packages/context-file/src/codec.ts @@ -77,7 +77,10 @@ export function assertPlainData(value: unknown, where = "value"): void { if (ancestors.has(entry)) throw new Error(`${path} is cyclic`); ancestors.add(entry); if (Array.isArray(entry)) { - entry.forEach((child, index) => walk(child, `${path}[${index}]`)); + for (let index = 0; index < entry.length; index += 1) { + if (!(index in entry)) throw new Error(`${path}[${index}] is undefined`); + walk(entry[index], `${path}[${index}]`); + } ancestors.delete(entry); return; } @@ -85,6 +88,9 @@ export function assertPlainData(value: unknown, where = "value"): void { if (prototype !== Object.prototype && prototype !== null) { throw new Error(`${path} is not a plain object`); } + if (Object.getOwnPropertySymbols(entry).length > 0) { + throw new Error(`${path} has symbol keys that JSON cannot represent`); + } for (const [key, child] of Object.entries(entry)) walk(child, `${path}.${key}`); ancestors.delete(entry); }; @@ -184,7 +190,12 @@ function asNumberRecord(value: unknown, where: string): Record { const record = asRecord(value, where); const out: Record = {}; for (const [key, entry] of Object.entries(record)) { - out[key] = asFiniteNumber(entry, `${where}.${key}`); + Object.defineProperty(out, key, { + value: asFiniteNumber(entry, `${where}.${key}`), + enumerable: true, + configurable: true, + writable: true, + }); } return out; } diff --git a/packages/context-file/src/index.test.ts b/packages/context-file/src/index.test.ts index 11e86a3..dde7a88 100644 --- a/packages/context-file/src/index.test.ts +++ b/packages/context-file/src/index.test.ts @@ -9,7 +9,7 @@ import { createInMemoryContextStore, type SemanticContextStoreContract, } from "@circe/context"; -import { createFileContextStore, type FileContextStore } from "./index"; +import { createFileContextStore, decodeContextLogOp, type FileContextStore } from "./index"; const NOW = "2026-09-24T09:00:00.000Z"; const LATER = "2026-09-24T09:05:00.000Z"; @@ -153,6 +153,38 @@ function buildRichStore(store: SemanticContextStoreContract) { } describe("durable semantic context store", () => { + it("preserves special probability keys while decoding", () => { + const trace = { + ...decisionTrace("decision:proto"), + probabilities: JSON.parse('{"__proto__":0.75,"yes":0.25}') as Record, + }; + const decoded = decodeContextLogOp({ op: "record-decision", trace }); + expect(decoded.op).toBe("record-decision"); + if (decoded.op !== "record-decision") throw new Error("unexpected decoded operation"); + expect(Object.hasOwn(decoded.trace.probabilities ?? {}, "__proto__")).toBe(true); + expect(decoded.trace.probabilities?.["__proto__"]).toBe(0.75); + }); + + it("applies the same decoded operation shape that replay will apply", () => { + const path = tempPath(); + const store = createFileContextStore(path); + store.putEntity({ + ref: a, + kind: "task", + firstSeenAt: NOW, + lastObservedAt: NOW, + observationIds: [], + runtimeOnly: "must be discarded consistently", + } as never); + const before = store.entity(a); + store.close(); + + const reopened = createFileContextStore(path); + expect(reopened.entity(a)).toEqual(before); + expect(reopened.entity(a)).not.toHaveProperty("runtimeOnly"); + reopened.close(); + }); + it("replays an exact state, including generations and the store version", async () => { const path = tempPath(); const first = createFileContextStore(path); @@ -277,6 +309,13 @@ describe("durable semantic context store", () => { const headerless = tempPath(); writeFileSync(headerless, '{"op":"invalidate","ids":[],"at":"2026-09-24T09:00:00.000Z"}\n'); expect(() => createFileContextStore(headerless)).toThrow(/does not start with a format record/); + + const repeatedHeader = tempPath(); + writeFileSync( + repeatedHeader, + '{"op":"format","name":"circe-semantic-store","version":1}\n{"op":"format","name":"circe-semantic-store","version":1}\n', + ); + expect(() => createFileContextStore(repeatedHeader)).toThrow(/unexpected format record at operation 2/); }); it("reports the position of a decodable operation that cannot be applied", () => { @@ -317,6 +356,16 @@ describe("durable semantic context store", () => { computedAt: NOW, }), ).toThrow(/finite number/); + const symbolKeyed = { value: 1, [Symbol("lost")]: 2 }; + expect(() => store.recordObservation({ + ref: a, + version: { authority: "tracker", token: "symbol", observedAt: NOW }, + attributes: symbolKeyed, + evidence: authorityEvidence("tracker", "symbol", NOW), + observedAt: NOW, + })).toThrow(/symbol keys/); + const sparse = new Array(1); + expect(() => store.invalidate(sparse, NOW)).toThrow(/undefined/); // The store is still usable: validation failed before the append boundary. observe(store, a, "a1"); diff --git a/packages/context-file/src/index.ts b/packages/context-file/src/index.ts index 9d85a33..2277b30 100644 --- a/packages/context-file/src/index.ts +++ b/packages/context-file/src/index.ts @@ -7,6 +7,7 @@ import { CONTEXT_LOG_FORMAT, assertPlainData, decodeContextLogOp, + encodeContextLogOp, type ContextLogOp, } from "./codec"; @@ -113,6 +114,9 @@ export function createFileContextStore( throw new Error(`semantic store ${path} does not start with a format record`); } for (const [index, op] of existing.entries()) { + if (index > 0 && op.op === "format") { + throw new Error(`semantic store ${path}: unexpected format record at operation ${index + 1}`); + } try { applyContextLogOp(inner, op); } catch (error) { @@ -125,41 +129,46 @@ export function createFileContextStore( } /** Validate representability, then make the operation durable, then apply. */ - const record = (op: ContextLogOp): void => { + const record = (op: T): T => { assertPlainData(op, "operation"); - log.append(op); + // Replay applies the decoder's normalized shape. Apply that same shape + // now, so unknown runtime fields and omitted optional values cannot make + // live state differ from a reopened store. + const canonical = decodeContextLogOp(JSON.parse(encodeContextLogOp(op))) as T; + log.append(canonical); + return canonical; }; return { version: () => inner.version(), putEntity(entity) { - record({ op: "put-entity", record: entity }); - inner.putEntity(entity); + const op = record({ op: "put-entity", record: entity }); + inner.putEntity(op.record); }, recordObservation(input) { - record({ op: "record-observation", input }); - return inner.recordObservation(input); + const op = record({ op: "record-observation", input }); + return inner.recordObservation(op.input); }, addRelation(input) { - record({ op: "add-relation", input }); - return inner.addRelation(input); + const op = record({ op: "add-relation", input }); + return inner.addRelation(op.input); }, recordEvent(input) { - record({ op: "record-event", input }); - return inner.recordEvent(input); + const op = record({ op: "record-event", input }); + return inner.recordEvent(op.input); }, recordAssertion(input) { - record({ op: "record-assertion", input }); - return inner.recordAssertion(input); + const op = record({ op: "record-assertion", input }); + return inner.recordAssertion(op.input); }, addAnnotation(input) { - record({ op: "add-annotation", input }); - return inner.addAnnotation(input); + const op = record({ op: "add-annotation", input }); + return inner.addAnnotation(op.input); }, addBinding(input) { - record({ op: "add-binding", input }); - return inner.addBinding(input); + const op = record({ op: "add-binding", input }); + return inner.addBinding(op.input); }, recordDecision(trace) { // Pre-check so a rejected duplicate never becomes a durable record that @@ -169,8 +178,8 @@ export function createFileContextStore( `duplicate decision id ${trace.id}: decision provenance is never overwritten`, ); } - record({ op: "record-decision", trace }); - return inner.recordDecision(trace); + const op = record({ op: "record-decision", trace }); + return inner.recordDecision(op.trace); }, recordFrame(frame) { const existingFrame = inner.frame(frame.id); @@ -180,27 +189,28 @@ export function createFileContextStore( `duplicate frame id ${frame.id} with different content: frames are immutable`, ); } - record({ op: "record-frame", frame }); - inner.recordFrame(frame); + const op = record({ op: "record-frame", frame }); + inner.recordFrame(op.frame); }, upsertSalience(input) { - record({ op: "upsert-salience", record: input }); - inner.upsertSalience(input); + const op = record({ op: "upsert-salience", record: input }); + inner.upsertSalience(op.record); }, invalidate(ids, at) { - record({ op: "invalidate", ids, at }); - return inner.invalidate(ids, at); + const op = record({ op: "invalidate", ids, at }); + return inner.invalidate(op.ids, op.at); }, invalidateDependents(ids, at) { - record({ op: "invalidate-dependents", ids, at }); - return inner.invalidateDependents(ids, at); + const op = record({ op: "invalidate-dependents", ids, at }); + return inner.invalidateDependents(op.ids, op.at); }, currentGeneration(ref) { // Minting a generation is a state transition: record it exactly once so // replay reproduces the same generation ids and dependency links. const existingGeneration = inner.peekGeneration(ref); if (existingGeneration === undefined) { - record({ op: "ensure-generation", ref }); + const op = record({ op: "ensure-generation", ref }); + return inner.currentGeneration(op.ref); } return inner.currentGeneration(ref); }, diff --git a/packages/file-log/src/index.test.ts b/packages/file-log/src/index.test.ts index 7caa801..f3159c8 100644 --- a/packages/file-log/src/index.test.ts +++ b/packages/file-log/src/index.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -73,6 +73,27 @@ describe("append-only file log", () => { expect(() => numberLog(directory)).toThrow(/not a regular file/); }); + it("rejects corrupt UTF-8 in a completed record instead of replacing bytes", () => { + const path = tempPath(); + writeFileSync(path, Buffer.concat([ + Buffer.from('{"n":1,"evidence":"'), Buffer.from([0xff]), Buffer.from('"}\n'), + ])); + expect(() => numberLog(path)).toThrow(/invalid UTF-8/); + }); + + it("repairs an unterminated UTF-8 fragment at the exact byte boundary", () => { + const path = tempPath(); + writeFileSync( + path, + Buffer.concat([Buffer.from('{"n":1}\n{"n":'), Buffer.from([0xe2, 0x82])]), + ); + const log = numberLog(path); + expect(log.read()).toEqual([1]); + log.append(2); + log.close(); + expect(readFileSync(path, "utf8")).toBe('{"n":1}\n{"n":2}\n'); + }); + it("uses the caller's label in corruption messages", () => { const path = tempPath(); writeFileSync(path, '{"n":1}\n{"unrelated":true}\n'); diff --git a/packages/file-log/src/index.ts b/packages/file-log/src/index.ts index 594d04d..c55cbcb 100644 --- a/packages/file-log/src/index.ts +++ b/packages/file-log/src/index.ts @@ -11,6 +11,8 @@ import { } from "node:fs"; import { dirname } from "node:path"; +const strictUtf8 = new TextDecoder("utf-8", { fatal: true }); + /** * Generic append-only JSONL file log. * @@ -76,22 +78,45 @@ function parseLog(path: string, decode: (value: unknown) => T, label: string) if (stats.isSymbolicLink() || !stats.isFile()) { throw new Error(`${label} path ${path} is not a regular file`); } - const text = readFileSync(path, "utf8"); - const endsWithNewline = text.endsWith("\n"); - const totalBytes = Buffer.byteLength(text, "utf8"); - const lines = text.split("\n"); + const bytes = readFileSync(path); + const endsWithNewline = bytes.length > 0 && bytes[bytes.length - 1] === 0x0a; + const totalBytes = bytes.length; + const lines: Buffer[] = []; + let lineStart = 0; + for (let index = 0; index < bytes.length; index += 1) { + if (bytes[index] === 0x0a) { + lines.push(bytes.subarray(lineStart, index)); + lineStart = index + 1; + } + } + lines.push(bytes.subarray(lineStart)); const lastIndex = lines.length - 1; let lastNonEmpty = -1; - lines.forEach((line, index) => { - if (line.trim()) lastNonEmpty = index; + const decodedLines: Array = lines.map((line, index) => { + try { + const decoded = strictUtf8.decode(line); + if (decoded.trim()) lastNonEmpty = index; + return decoded; + } catch (error) { + if (index === lastIndex && !endsWithNewline) { + lastNonEmpty = index; + return undefined; + } + throw new Error(`corrupt ${label} line ${index + 1} in ${path}: invalid UTF-8 (${String(error)})`); + } }); const values: T[] = []; let offset = 0; let lastRecordHadNewline = false; - for (const [index, raw] of lines.entries()) { + for (const [index, rawBytes] of lines.entries()) { const isLastSegment = index === lastIndex; const hasNewline = !(isLastSegment && !endsWithNewline); + const raw = decodedLines[index]; + if (raw === undefined) { + // An incomplete UTF-8 sequence at the tail is a torn append. + return { values, repair: { offset, needsSeparator: false } }; + } const trimmed = raw.trim(); if (!trimmed) { // only the final segment produced by a trailing newline may be empty @@ -101,7 +126,7 @@ function parseLog(path: string, decode: (value: unknown) => T, label: string) try { const parsed: unknown = JSON.parse(trimmed); values.push(decode(parsed)); - offset += Buffer.byteLength(raw, "utf8") + (hasNewline ? 1 : 0); + offset += rawBytes.length + (hasNewline ? 1 : 0); lastRecordHadNewline = hasNewline; } catch (error) { // Only a torn JSON fragment at the final line is a crash artifact; diff --git a/packages/perception-jev/README.md b/packages/perception-jev/README.md new file mode 100644 index 0000000..e79df91 --- /dev/null +++ b/packages/perception-jev/README.md @@ -0,0 +1,26 @@ +# `@circe/perception-jev` + +This package contains the read-only perception backend seam and the optional +TypeSafe Jev HTTP client. It does not execute capabilities or mutate Circe +state. + +`createTypeSafeJevClient()` uses the official `POST /v1/systemone` API with +native `fetch`, so it adds no provider SDK dependency. It reads +`TYPESAFE_API_KEY` when a request runs; callers can instead pass `apiKey`, a +custom `baseUrl`, `timeoutMs`, and a mocked `fetch` implementation. The caller +supplies the model name and should choose one enabled for its account. This +adapter does not discover or silently substitute models. + +Each provider call has a 30-second timeout by default; configure `timeoutMs` +to change it. The timeout covers both response headers and the response body. + +The adapter maps Circe `binary`, `choice`, and `score` questions to TypeSafe +`noul`, `choice`, and `score` questions, then translates named answers back to +the Circe contract. Missing or malformed answers, HTTP failures, and timeouts +reject the backend call so the workflow records a backend failure. + +The scripted client is a deterministic test double. Its results are not live +semantic model evidence and must not be reported as live provider evaluation. + +Official API contract: [TypeSafe OpenAPI](https://api.typesafe.ai/openapi.json) +and [API reference](https://api.typesafe.ai/docs). diff --git a/packages/perception-jev/src/backend.ts b/packages/perception-jev/src/backend.ts index 87c8c54..1a46f7f 100644 --- a/packages/perception-jev/src/backend.ts +++ b/packages/perception-jev/src/backend.ts @@ -131,16 +131,22 @@ export function createJevBackend(options: { // Status is never defaulted: an absent or malformed discriminator must // fail validation rather than silently mean RESOLVED. + // Capture metadata once: untrusted clients can return getters that vary + // between reads, so validation and trace construction must see one value. const rawStatus = response.status as PerceptionStatus | undefined; + const rawModel = response.model; + const rawLatencyMs = response.latencyMs; + const rawAnswers = response.answers; + const answeredModel = typeof rawModel === "string" ? rawModel : options.model; const latencyMs = - typeof response.latencyMs === "number" && Number.isFinite(response.latencyMs) - ? response.latencyMs + typeof rawLatencyMs === "number" && Number.isFinite(rawLatencyMs) + ? rawLatencyMs : Date.now() - startedAt; const analysis = analyzePerceptionResult(specs, { status: rawStatus, - answers: response.answers, + answers: rawAnswers, backend: id, - model: options.model, + model: answeredModel, latencyMs, }); if (!analysis.decoded || analysis.issues.length > 0) { diff --git a/packages/perception-jev/src/index.ts b/packages/perception-jev/src/index.ts index 51710dd..fd5132e 100644 --- a/packages/perception-jev/src/index.ts +++ b/packages/perception-jev/src/index.ts @@ -1,3 +1,4 @@ export * from "./backend"; export * from "./evaluation"; export * from "./scripted-client"; +export * from "./typesafe-client"; diff --git a/packages/perception-jev/src/perception-jev.test.ts b/packages/perception-jev/src/perception-jev.test.ts index 3b7c6b7..e9526a3 100644 --- a/packages/perception-jev/src/perception-jev.test.ts +++ b/packages/perception-jev/src/perception-jev.test.ts @@ -16,6 +16,7 @@ import { runReadOnlyEvaluation, type JevDecisionClient, type JevDecisionRequest, + createTypeSafeJevClient, } from "./index"; const now = "2026-09-22T09:00:00.000Z"; @@ -63,6 +64,156 @@ const choiceSpec: DecisionSpec = { const workflow = { id: "read-only.v1", stages: [{ id: "referent", specs: () => [choiceSpec] }] }; describe("jev backend seam", () => { + it("maps Circe decisions to the official TypeSafe HTTP contract", async () => { + let captured: { url: string; init?: RequestInit } | undefined; + const client = createTypeSafeJevClient({ + apiKey: "test-key", + baseUrl: "https://typesafe.test/", + fetch: async (input, init) => { + captured = { url: String(input), init }; + return new Response(JSON.stringify({ + model: "jev-account-model", + answers: { + referent: { type: "choice", choice: "B", confidence: 0.95, probabilities: { A: 0.02, B: 0.97, none: 0.01 } }, + }, + usage: { input_tokens: 3, output_tokens: 1 }, + }), { status: 200, headers: { "content-type": "application/json" } }); + }, + }); + const backend = createJevBackend({ client, model: "jev-account-model" }); + const frame = await frameFor(); + const run = await runPerceptionWorkflow({ backend, frame, workflow, policy: ACCEPTANCE_POLICY_V1, now }); + expect(captured?.url).toBe("https://typesafe.test/v1/systemone"); + expect(new Headers(captured?.init?.headers).get("authorization")).toBe("Bearer test-key"); + const body = JSON.parse(String(captured?.init?.body)); + expect(body).toMatchObject({ + model: "jev-account-model", + state: { utterance: { text: "which of these?" }, candidates: frame.candidates }, + questions: { referent: { type: "choice", instructions: choiceSpec.question, criteria: { A: "A", B: "B", none: "none" } } }, + }); + expect(run.status).toBe("RESOLVED"); + expect(run.traces[0]?.model).toBe("jev-account-model"); + }); + + it("fails closed on HTTP errors and missing keyed answers", async () => { + const frame = await frameFor(); + const failed = createJevBackend({ + client: createTypeSafeJevClient({ apiKey: "test-key", fetch: async () => new Response("", { status: 503 }) }), + model: "jev-account-model", + }); + await expect(failed.decide(frame, [choiceSpec])).rejects.toThrow("HTTP 503"); + + const missing = createJevBackend({ + client: createTypeSafeJevClient({ + apiKey: "test-key", + fetch: async () => new Response(JSON.stringify({ model: "jev-account-model", answers: {}, usage: { input_tokens: 1, output_tokens: 1 } }), { status: 200 }), + }), + model: "jev-account-model", + }); + await expect(missing.decide(frame, [choiceSpec])).rejects.toThrow("omitted answer"); + }); + + it("rejects answers for decision ids that were not requested", async () => { + const client = createTypeSafeJevClient({ + apiKey: "test-key", + fetch: async () => new Response(JSON.stringify({ + model: "jev-account-model", + answers: { + referent: { type: "choice", choice: "B", confidence: 0.95, probabilities: { A: 0.02, B: 0.97, none: 0.01 } }, + unrelated: { type: "noul", noul: 0.99 }, + }, + usage: { input_tokens: 3, output_tokens: 2 }, + }), { status: 200 }), + }); + await expect(createJevBackend({ client, model: "jev-account-model" }).decide(await frameFor(), [choiceSpec])) + .rejects.toThrow("unasked answer for decision unrelated"); + }); + + it("rejects missing and unexpected score probability levels", async () => { + const frame = await frameFor(); + const scoreSpec: DecisionSpec = { id: "severity", kind: "score", decision: "risk", question: "How severe?", levels: ["low", "medium", "high"] }; + for (const probabilities of [ + { "0": 0.1, "1": 0.1, "2": 0.8, "3": 0 }, + { "0": 0.1, "1": 0.1 }, + ]) { + const client = createTypeSafeJevClient({ + apiKey: "test-key", + fetch: async () => new Response(JSON.stringify({ + model: "jev-account-model", + answers: { severity: { type: "score", score: 1.7, confidence: 0.9, legend: {}, probabilities } }, + usage: { input_tokens: 3, output_tokens: 1 }, + }), { status: 200 }), + }); + await expect(createJevBackend({ client, model: "jev-account-model" }).decide(frame, [scoreSpec])) + .rejects.toThrow(probabilities["3"] === undefined ? "omitted a score probability level" : "unexpected score probability level"); + } + }); + + it("turns an aborted TypeSafe request into a bounded timeout failure", async () => { + const client = createTypeSafeJevClient({ + apiKey: "test-key", + timeoutMs: 1, + fetch: async (_input, init) => ({ + ok: true, + json: () => new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => reject(new Error("aborted body")), { once: true }); + }), + }) as Response, + }); + await expect(createJevBackend({ client, model: "jev-account-model" }).decide(await frameFor(), [choiceSpec])) + .rejects.toThrow("timed out"); + }); + + it("reads provider metadata getters once before validation and tracing", async () => { + const frame = await frameFor(); + const reads = { status: 0, model: 0, latency: 0, answers: 0 }; + const response = { + get status() { reads.status += 1; return "RESOLVED"; }, + get model() { reads.model += 1; return "jev-returned"; }, + get latencyMs() { reads.latency += 1; return 17; }, + get answers() { + reads.answers += 1; + return [{ id: "referent", kind: "choice", choice: "B", probabilities: { A: 0.02, B: 0.97, none: 0.01 }, confidence: 0.95 }]; + }, + }; + const result = await createJevBackend({ client: { id: "getter-metadata", async complete() { return response; } }, model: "jev-requested" }) + .decide(frame, [choiceSpec]); + expect(reads).toEqual({ status: 1, model: 1, latency: 1, answers: 1 }); + expect(result.model).toBe("jev-returned"); + expect(result.latencyMs).toBe(17); + }); + + it("maps binary and score questions and preserves their answer distributions", async () => { + let body: any; + const client = createTypeSafeJevClient({ + apiKey: "test-key", + fetch: async (_input, init) => { + body = JSON.parse(String(init?.body)); + return new Response(JSON.stringify({ + model: "jev-account-model", + answers: { + urgent: { type: "noul", noul: 0.8 }, + severity: { type: "score", score: 1.7, confidence: 0.9, legend: { "0": "low", "1": "medium", "2": "high" }, probabilities: { "0": 0.1, "1": 0.1, "2": 0.8 } }, + }, + usage: { input_tokens: 3, output_tokens: 2 }, + }), { status: 200 }); + }, + }); + const specs: DecisionSpec[] = [ + { ...choiceSpec, id: "urgent", kind: "binary", decision: "risk", criteria: { true: "urgent", false: "not urgent" } }, + { ...choiceSpec, id: "severity", kind: "score", decision: "risk", levels: ["low", "medium", "high"] }, + ]; + const result = await createJevBackend({ client, model: "jev-account-model" }).decide(await frameFor(), specs); + expect(body.questions).toEqual({ + urgent: { type: "noul", instructions: choiceSpec.question, criteria: { true: "urgent", false: "not urgent" } }, + severity: { type: "score", instructions: choiceSpec.question, criteria: ["low", "medium", "high"] }, + }); + expect(result.answers).toEqual([ + { id: "urgent", kind: "binary", probability: 0.8 }, + { id: "severity", kind: "score", score: 1.7, probabilities: [0.1, 0.1, 0.8], confidence: 0.9 }, + ]); + }); + it("translates decisions into a canonical request and decodes the response", async () => { const requests: JevDecisionRequest[] = []; const client: JevDecisionClient = { diff --git a/packages/perception-jev/src/typesafe-client.ts b/packages/perception-jev/src/typesafe-client.ts new file mode 100644 index 0000000..f5aab18 --- /dev/null +++ b/packages/perception-jev/src/typesafe-client.ts @@ -0,0 +1,126 @@ +import type { DecisionSpec } from "@circe/perception"; +import type { JevDecisionClient, JevDecisionRequest } from "./backend"; + +const DEFAULT_BASE_URL = "https://api.typesafe.ai"; +const DEFAULT_TIMEOUT_MS = 30_000; + +export interface TypeSafeJevClientOptions { + apiKey?: string; + apiKeyEnv?: string; + baseUrl?: string; + timeoutMs?: number; + fetch?: typeof globalThis.fetch; +} + +type TypeSafeQuestion = + | { type: "noul"; instructions: string; criteria?: { true?: string; false?: string } } + | { type: "choice"; instructions: string; criteria: Record } + | { type: "score"; instructions: string; criteria: string[] }; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function questionFor(spec: DecisionSpec): TypeSafeQuestion { + switch (spec.kind) { + case "binary": + return { type: "noul", instructions: spec.question, ...(spec.criteria ? { criteria: spec.criteria } : {}) }; + case "choice": + return { + type: "choice", + instructions: spec.question, + criteria: Object.fromEntries(spec.options.map(({ id, description }) => [id, description ?? id])), + }; + case "score": + return { type: "score", instructions: spec.question, criteria: [...spec.levels] }; + } +} + +function answerFor(spec: DecisionSpec, raw: unknown): Record { + if (!isRecord(raw)) throw new Error(`TypeSafe returned a malformed answer for decision ${spec.id}`); + if (spec.kind === "binary" && raw.type === "noul" && typeof raw.noul === "number") { + return { id: spec.id, kind: "binary", probability: raw.noul }; + } + if (spec.kind === "choice" && raw.type === "choice" && typeof raw.choice === "string" && isRecord(raw.probabilities) && typeof raw.confidence === "number") { + return { id: spec.id, kind: "choice", choice: raw.choice, probabilities: raw.probabilities, confidence: raw.confidence }; + } + if (spec.kind === "score" && raw.type === "score" && typeof raw.score === "number" && isRecord(raw.probabilities) && typeof raw.confidence === "number") { + const providerProbabilities = raw.probabilities; + const expectedLevels = spec.levels.map((_, index) => String(index)); + const unexpectedLevel = Object.keys(providerProbabilities).find((level) => !expectedLevels.includes(level)); + if (unexpectedLevel !== undefined) { + throw new Error(`TypeSafe returned unexpected score probability level ${unexpectedLevel} for decision ${spec.id}`); + } + const probabilities = expectedLevels.map((level) => providerProbabilities[level]); + if (probabilities.some((probability) => typeof probability !== "number")) { + throw new Error(`TypeSafe omitted a score probability level for decision ${spec.id}`); + } + return { id: spec.id, kind: "score", score: raw.score, probabilities, confidence: raw.confidence }; + } + throw new Error(`TypeSafe answer type did not match decision ${spec.id}`); +} + +/** + * Native-fetch adapter for TypeSafe's documented System One HTTP API. + * Credentials are read only when complete() runs; callers may supply an + * explicit key for managed hosts and tests. + */ +export function createTypeSafeJevClient(options: TypeSafeJevClientOptions = {}): JevDecisionClient { + const fetchImpl = options.fetch ?? globalThis.fetch; + const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, ""); + const keyName = options.apiKeyEnv ?? "TYPESAFE_API_KEY"; + return { + id: "typesafe-jev", + async complete(request: JevDecisionRequest): Promise { + const apiKey = options.apiKey ?? process.env[keyName]; + if (!apiKey) throw new Error(`TypeSafe API key is required (set ${keyName})`); + const questions = Object.fromEntries(request.decisions.map((spec) => [spec.id, questionFor(spec)])); + if (Object.keys(questions).length !== request.decisions.length) { + throw new Error("TypeSafe request contains duplicate decision ids"); + } + const startedAt = Date.now(); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), options.timeoutMs ?? DEFAULT_TIMEOUT_MS); + try { + const response = await fetchImpl(`${baseUrl}/v1/systemone`, { + method: "POST", + headers: { authorization: `Bearer ${apiKey}`, "content-type": "application/json" }, + body: JSON.stringify({ + model: request.model, + state: { + utterance: request.context.utterance, + interaction: request.context.interaction, + candidates: request.context.candidates, + capabilities: request.context.capabilities, + coverage: request.coverage, + }, + questions, + }), + signal: controller.signal, + }); + if (!response.ok) throw new Error(`TypeSafe request failed with HTTP ${response.status}`); + const payload: unknown = await response.json(); + if (!isRecord(payload) || !isRecord(payload.answers)) throw new Error("TypeSafe returned a malformed response"); + const providerAnswers = payload.answers; + const askedIds = new Set(request.decisions.map((spec) => spec.id)); + const unexpectedId = Object.keys(providerAnswers).find((id) => !askedIds.has(id)); + if (unexpectedId !== undefined) throw new Error(`TypeSafe returned unasked answer for decision ${unexpectedId}`); + const answers = request.decisions.map((spec) => { + if (!Object.hasOwn(providerAnswers, spec.id)) throw new Error(`TypeSafe omitted answer for decision ${spec.id}`); + return answerFor(spec, providerAnswers[spec.id]); + }); + return { + status: "RESOLVED", + answers, + ...(typeof payload.model === "string" ? { model: payload.model } : {}), + latencyMs: Date.now() - startedAt, + }; + } catch (error) { + if (controller.signal.aborted) throw new Error("TypeSafe request timed out"); + throw error; + } finally { + clearTimeout(timeout); + } + }, + }; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6a95ea4..3de1510 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,6 +29,9 @@ importers: '@circe/evals': specifier: workspace:* version: link:../../packages/evals + '@circe/file-log': + specifier: workspace:* + version: link:../../packages/file-log '@circe/journal-file': specifier: workspace:* version: link:../../packages/journal-file