session-entry-stepper.test.ts 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821
  1. import { describe, expect, test } from "bun:test"
  2. import * as DateTime from "effect/DateTime"
  3. import * as FastCheck from "effect/testing/FastCheck"
  4. import { SessionEntry } from "../../src/v2/session-entry"
  5. import { SessionEntryStepper } from "../../src/v2/session-entry-stepper"
  6. import { SessionEvent } from "../../src/v2/session-event"
  7. const time = (n: number) => DateTime.makeUnsafe(n)
  8. const word = FastCheck.string({ minLength: 1, maxLength: 8 })
  9. const text = FastCheck.string({ maxLength: 16 })
  10. const texts = FastCheck.array(text, { maxLength: 8 })
  11. const val = FastCheck.oneof(FastCheck.boolean(), FastCheck.integer(), FastCheck.string({ maxLength: 12 }))
  12. const dict = FastCheck.dictionary(word, val, { maxKeys: 4 })
  13. const files = FastCheck.array(
  14. word.map((x) => SessionEvent.FileAttachment.create({ uri: `file://${encodeURIComponent(x)}`, mime: "text/plain" })),
  15. { maxLength: 2 },
  16. )
  17. function maybe<A>(arb: FastCheck.Arbitrary<A>) {
  18. return FastCheck.oneof(FastCheck.constant(undefined), arb)
  19. }
  20. function assistant() {
  21. return new SessionEntry.Assistant({
  22. id: SessionEvent.ID.create(),
  23. type: "assistant",
  24. time: { created: time(0) },
  25. content: [],
  26. })
  27. }
  28. function memoryState() {
  29. const state: SessionEntryStepper.MemoryState = {
  30. entries: [],
  31. pending: [],
  32. }
  33. return state
  34. }
  35. function active() {
  36. const state: SessionEntryStepper.MemoryState = {
  37. entries: [assistant()],
  38. pending: [],
  39. }
  40. return state
  41. }
  42. function run(events: SessionEvent.Event[], state = memoryState()) {
  43. return events.reduce<SessionEntryStepper.MemoryState>((state, event) => SessionEntryStepper.step(state, event), state)
  44. }
  45. function last(state: SessionEntryStepper.MemoryState) {
  46. const entry = [...state.pending, ...state.entries].reverse().find((x) => x.type === "assistant")
  47. expect(entry?.type).toBe("assistant")
  48. return entry?.type === "assistant" ? entry : undefined
  49. }
  50. function texts_of(state: SessionEntryStepper.MemoryState) {
  51. const entry = last(state)
  52. if (!entry) return []
  53. return entry.content.filter((x): x is SessionEntry.AssistantText => x.type === "text")
  54. }
  55. function reasons(state: SessionEntryStepper.MemoryState) {
  56. const entry = last(state)
  57. if (!entry) return []
  58. return entry.content.filter((x): x is SessionEntry.AssistantReasoning => x.type === "reasoning")
  59. }
  60. function tools(state: SessionEntryStepper.MemoryState) {
  61. const entry = last(state)
  62. if (!entry) return []
  63. return entry.content.filter((x): x is SessionEntry.AssistantTool => x.type === "tool")
  64. }
  65. function tool(state: SessionEntryStepper.MemoryState, callID: string) {
  66. return tools(state).find((x) => x.callID === callID)
  67. }
  68. function adapterStore() {
  69. return {
  70. committed: [] as SessionEntry.Entry[],
  71. deferred: [] as SessionEntry.Entry[],
  72. }
  73. }
  74. function adapterFor(store: ReturnType<typeof adapterStore>): SessionEntryStepper.Adapter<typeof store> {
  75. const activeAssistantIndex = () =>
  76. store.committed.findLastIndex((entry) => entry.type === "assistant" && !entry.time.completed)
  77. const getCurrentAssistant = () => {
  78. const index = activeAssistantIndex()
  79. if (index < 0) return
  80. const assistant = store.committed[index]
  81. return assistant?.type === "assistant" ? assistant : undefined
  82. }
  83. return {
  84. getCurrentAssistant,
  85. updateAssistant(assistant) {
  86. const index = activeAssistantIndex()
  87. if (index < 0) return
  88. const current = store.committed[index]
  89. if (current?.type !== "assistant") return
  90. store.committed[index] = assistant
  91. },
  92. appendEntry(entry) {
  93. store.committed.push(entry)
  94. },
  95. appendPending(entry) {
  96. store.deferred.push(entry)
  97. },
  98. finish() {
  99. return store
  100. },
  101. }
  102. }
  103. describe("session-entry-stepper", () => {
  104. describe("stepWith", () => {
  105. test("reduces through a custom adapter", () => {
  106. const store = adapterStore()
  107. store.committed.push(assistant())
  108. SessionEntryStepper.stepWith(adapterFor(store), SessionEvent.Prompt.create({ text: "hello", timestamp: time(1) }))
  109. SessionEntryStepper.stepWith(adapterFor(store), SessionEvent.Reasoning.Started.create({ timestamp: time(2) }))
  110. SessionEntryStepper.stepWith(adapterFor(store), SessionEvent.Reasoning.Delta.create({ delta: "thinking", timestamp: time(3) }))
  111. SessionEntryStepper.stepWith(adapterFor(store), SessionEvent.Reasoning.Ended.create({ text: "thought", timestamp: time(4) }))
  112. SessionEntryStepper.stepWith(adapterFor(store), SessionEvent.Text.Started.create({ timestamp: time(5) }))
  113. SessionEntryStepper.stepWith(adapterFor(store), SessionEvent.Text.Delta.create({ delta: "world", timestamp: time(6) }))
  114. SessionEntryStepper.stepWith(
  115. adapterFor(store),
  116. SessionEvent.Step.Ended.create({
  117. reason: "stop",
  118. cost: 1,
  119. tokens: {
  120. input: 1,
  121. output: 2,
  122. reasoning: 3,
  123. cache: {
  124. read: 4,
  125. write: 5,
  126. },
  127. },
  128. timestamp: time(7),
  129. }),
  130. )
  131. expect(store.deferred).toHaveLength(1)
  132. expect(store.deferred[0]?.type).toBe("user")
  133. expect(store.committed).toHaveLength(1)
  134. expect(store.committed[0]?.type).toBe("assistant")
  135. if (store.committed[0]?.type !== "assistant") return
  136. expect(store.committed[0].content).toEqual([
  137. { type: "reasoning", text: "thought" },
  138. { type: "text", text: "world" },
  139. ])
  140. expect(store.committed[0].time.completed).toEqual(time(7))
  141. })
  142. })
  143. describe("memory", () => {
  144. test("tracks and replaces the current assistant", () => {
  145. const state = active()
  146. const adapter = SessionEntryStepper.memory(state)
  147. const current = adapter.getCurrentAssistant()
  148. expect(current?.type).toBe("assistant")
  149. if (!current) return
  150. adapter.updateAssistant(
  151. new SessionEntry.Assistant({
  152. ...current,
  153. content: [new SessionEntry.AssistantText({ type: "text", text: "done" })],
  154. time: {
  155. ...current.time,
  156. completed: time(1),
  157. },
  158. }),
  159. )
  160. expect(adapter.getCurrentAssistant()).toBeUndefined()
  161. expect(state.entries[0]?.type).toBe("assistant")
  162. if (state.entries[0]?.type !== "assistant") return
  163. expect(state.entries[0].content).toEqual([{ type: "text", text: "done" }])
  164. expect(state.entries[0].time.completed).toEqual(time(1))
  165. })
  166. test("appends committed and pending entries", () => {
  167. const state = memoryState()
  168. const adapter = SessionEntryStepper.memory(state)
  169. const committed = SessionEntry.User.fromEvent(SessionEvent.Prompt.create({ text: "committed", timestamp: time(1) }))
  170. const pending = SessionEntry.User.fromEvent(SessionEvent.Prompt.create({ text: "pending", timestamp: time(2) }))
  171. adapter.appendEntry(committed)
  172. adapter.appendPending(pending)
  173. expect(state.entries).toEqual([committed])
  174. expect(state.pending).toEqual([pending])
  175. })
  176. test("stepWith through memory records reasoning", () => {
  177. const state = active()
  178. SessionEntryStepper.stepWith(SessionEntryStepper.memory(state), SessionEvent.Reasoning.Started.create({ timestamp: time(1) }))
  179. SessionEntryStepper.stepWith(SessionEntryStepper.memory(state), SessionEvent.Reasoning.Delta.create({ delta: "draft", timestamp: time(2) }))
  180. SessionEntryStepper.stepWith(
  181. SessionEntryStepper.memory(state),
  182. SessionEvent.Reasoning.Ended.create({ text: "final", timestamp: time(3) }),
  183. )
  184. expect(reasons(state)).toEqual([{ type: "reasoning", text: "final" }])
  185. })
  186. })
  187. describe("step", () => {
  188. describe("seeded pending assistant", () => {
  189. test("stores prompts in entries when no assistant is pending", () => {
  190. FastCheck.assert(
  191. FastCheck.property(word, (body) => {
  192. const next = SessionEntryStepper.step(memoryState(), SessionEvent.Prompt.create({ text: body, timestamp: time(1) }))
  193. expect(next.entries).toHaveLength(1)
  194. expect(next.entries[0]?.type).toBe("user")
  195. if (next.entries[0]?.type !== "user") return
  196. expect(next.entries[0].text).toBe(body)
  197. }),
  198. { numRuns: 50 },
  199. )
  200. })
  201. test("stores prompts in pending when an assistant is pending", () => {
  202. FastCheck.assert(
  203. FastCheck.property(word, (body) => {
  204. const next = SessionEntryStepper.step(active(), SessionEvent.Prompt.create({ text: body, timestamp: time(1) }))
  205. expect(next.pending).toHaveLength(1)
  206. expect(next.pending[0]?.type).toBe("user")
  207. if (next.pending[0]?.type !== "user") return
  208. expect(next.pending[0].text).toBe(body)
  209. }),
  210. { numRuns: 50 },
  211. )
  212. })
  213. test("accumulates text deltas on the latest text part", () => {
  214. FastCheck.assert(
  215. FastCheck.property(texts, (parts) => {
  216. const next = parts.reduce(
  217. (state, part, i) =>
  218. SessionEntryStepper.step(state, SessionEvent.Text.Delta.create({ delta: part, timestamp: time(i + 2) })),
  219. SessionEntryStepper.step(active(), SessionEvent.Text.Started.create({ timestamp: time(1) })),
  220. )
  221. expect(texts_of(next)).toEqual([
  222. {
  223. type: "text",
  224. text: parts.join(""),
  225. },
  226. ])
  227. }),
  228. { numRuns: 100 },
  229. )
  230. })
  231. test("routes later text deltas to the latest text segment", () => {
  232. FastCheck.assert(
  233. FastCheck.property(texts, texts, (a, b) => {
  234. const next = run(
  235. [
  236. SessionEvent.Text.Started.create({ timestamp: time(1) }),
  237. ...a.map((x, i) => SessionEvent.Text.Delta.create({ delta: x, timestamp: time(i + 2) })),
  238. SessionEvent.Text.Started.create({ timestamp: time(a.length + 2) }),
  239. ...b.map((x, i) => SessionEvent.Text.Delta.create({ delta: x, timestamp: time(i + a.length + 3) })),
  240. ],
  241. active(),
  242. )
  243. expect(texts_of(next)).toEqual([
  244. { type: "text", text: a.join("") },
  245. { type: "text", text: b.join("") },
  246. ])
  247. }),
  248. { numRuns: 50 },
  249. )
  250. })
  251. test("reasoning.ended replaces buffered reasoning text", () => {
  252. FastCheck.assert(
  253. FastCheck.property(texts, text, (parts, end) => {
  254. const next = run(
  255. [
  256. SessionEvent.Reasoning.Started.create({ timestamp: time(1) }),
  257. ...parts.map((x, i) => SessionEvent.Reasoning.Delta.create({ delta: x, timestamp: time(i + 2) })),
  258. SessionEvent.Reasoning.Ended.create({ text: end, timestamp: time(parts.length + 2) }),
  259. ],
  260. active(),
  261. )
  262. expect(reasons(next)).toEqual([
  263. {
  264. type: "reasoning",
  265. text: end,
  266. },
  267. ])
  268. }),
  269. { numRuns: 100 },
  270. )
  271. })
  272. test("tool.success completes the latest running tool", () => {
  273. FastCheck.assert(
  274. FastCheck.property(
  275. word,
  276. word,
  277. dict,
  278. maybe(text),
  279. maybe(dict),
  280. maybe(files),
  281. texts,
  282. (callID, title, input, output, metadata, attachments, parts) => {
  283. const next = run(
  284. [
  285. SessionEvent.Tool.Input.Started.create({ callID, name: "bash", timestamp: time(1) }),
  286. ...parts.map((x, i) =>
  287. SessionEvent.Tool.Input.Delta.create({ callID, delta: x, timestamp: time(i + 2) }),
  288. ),
  289. SessionEvent.Tool.Called.create({
  290. callID,
  291. tool: "bash",
  292. input,
  293. provider: { executed: true },
  294. timestamp: time(parts.length + 2),
  295. }),
  296. SessionEvent.Tool.Success.create({
  297. callID,
  298. title,
  299. output,
  300. metadata,
  301. attachments,
  302. provider: { executed: true },
  303. timestamp: time(parts.length + 3),
  304. }),
  305. ],
  306. active(),
  307. )
  308. const match = tool(next, callID)
  309. expect(match?.state.status).toBe("completed")
  310. if (match?.state.status !== "completed") return
  311. expect(match.time.ran).toEqual(time(parts.length + 2))
  312. expect(match.state.input).toEqual(input)
  313. expect(match.state.output).toBe(output ?? "")
  314. expect(match.state.title).toBe(title)
  315. expect(match.state.metadata).toEqual(metadata ?? {})
  316. expect(match.state.attachments).toEqual(attachments ?? [])
  317. },
  318. ),
  319. { numRuns: 50 },
  320. )
  321. })
  322. test("tool.error completes the latest running tool with an error", () => {
  323. FastCheck.assert(
  324. FastCheck.property(word, dict, word, maybe(dict), (callID, input, error, metadata) => {
  325. const next = run(
  326. [
  327. SessionEvent.Tool.Input.Started.create({ callID, name: "bash", timestamp: time(1) }),
  328. SessionEvent.Tool.Called.create({
  329. callID,
  330. tool: "bash",
  331. input,
  332. provider: { executed: true },
  333. timestamp: time(2),
  334. }),
  335. SessionEvent.Tool.Error.create({
  336. callID,
  337. error,
  338. metadata,
  339. provider: { executed: true },
  340. timestamp: time(3),
  341. }),
  342. ],
  343. active(),
  344. )
  345. const match = tool(next, callID)
  346. expect(match?.state.status).toBe("error")
  347. if (match?.state.status !== "error") return
  348. expect(match.time.ran).toEqual(time(2))
  349. expect(match.state.input).toEqual(input)
  350. expect(match.state.error).toBe(error)
  351. expect(match.state.metadata).toEqual(metadata ?? {})
  352. }),
  353. { numRuns: 50 },
  354. )
  355. })
  356. test("tool.success is ignored before tool.called promotes the tool to running", () => {
  357. FastCheck.assert(
  358. FastCheck.property(word, word, (callID, title) => {
  359. const next = run(
  360. [
  361. SessionEvent.Tool.Input.Started.create({ callID, name: "bash", timestamp: time(1) }),
  362. SessionEvent.Tool.Success.create({
  363. callID,
  364. title,
  365. provider: { executed: true },
  366. timestamp: time(2),
  367. }),
  368. ],
  369. active(),
  370. )
  371. const match = tool(next, callID)
  372. expect(match?.state).toEqual({
  373. status: "pending",
  374. input: "",
  375. })
  376. }),
  377. { numRuns: 50 },
  378. )
  379. })
  380. test("step.ended copies completion fields onto the pending assistant", () => {
  381. FastCheck.assert(
  382. FastCheck.property(FastCheck.integer({ min: 1, max: 1000 }), (n) => {
  383. const event = SessionEvent.Step.Ended.create({
  384. reason: "stop",
  385. cost: 1,
  386. tokens: {
  387. input: 1,
  388. output: 2,
  389. reasoning: 3,
  390. cache: {
  391. read: 4,
  392. write: 5,
  393. },
  394. },
  395. timestamp: time(n),
  396. })
  397. const next = SessionEntryStepper.step(active(), event)
  398. const entry = last(next)
  399. expect(entry).toBeDefined()
  400. if (!entry) return
  401. expect(entry.time.completed).toEqual(event.timestamp)
  402. expect(entry.cost).toBe(event.cost)
  403. expect(entry.tokens).toEqual(event.tokens)
  404. }),
  405. { numRuns: 50 },
  406. )
  407. })
  408. })
  409. describe("known reducer gaps", () => {
  410. test("prompt appends immutably when no assistant is pending", () => {
  411. FastCheck.assert(
  412. FastCheck.property(word, (body) => {
  413. const old = memoryState()
  414. const next = SessionEntryStepper.step(old, SessionEvent.Prompt.create({ text: body, timestamp: time(1) }))
  415. expect(old).not.toBe(next)
  416. expect(old.entries).toHaveLength(0)
  417. expect(next.entries).toHaveLength(1)
  418. }),
  419. { numRuns: 50 },
  420. )
  421. })
  422. test("prompt appends immutably when an assistant is pending", () => {
  423. FastCheck.assert(
  424. FastCheck.property(word, (body) => {
  425. const old = active()
  426. const next = SessionEntryStepper.step(old, SessionEvent.Prompt.create({ text: body, timestamp: time(1) }))
  427. expect(old).not.toBe(next)
  428. expect(old.pending).toHaveLength(0)
  429. expect(next.pending).toHaveLength(1)
  430. }),
  431. { numRuns: 50 },
  432. )
  433. })
  434. test("step.started creates an assistant consumed by follow-up events", () => {
  435. FastCheck.assert(
  436. FastCheck.property(texts, (parts) => {
  437. const next = run([
  438. SessionEvent.Step.Started.create({
  439. model: {
  440. id: "model",
  441. providerID: "provider",
  442. },
  443. timestamp: time(1),
  444. }),
  445. SessionEvent.Text.Started.create({ timestamp: time(2) }),
  446. ...parts.map((x, i) => SessionEvent.Text.Delta.create({ delta: x, timestamp: time(i + 3) })),
  447. SessionEvent.Step.Ended.create({
  448. reason: "stop",
  449. cost: 1,
  450. tokens: {
  451. input: 1,
  452. output: 2,
  453. reasoning: 3,
  454. cache: {
  455. read: 4,
  456. write: 5,
  457. },
  458. },
  459. timestamp: time(parts.length + 3),
  460. }),
  461. ])
  462. const entry = last(next)
  463. expect(entry).toBeDefined()
  464. if (!entry) return
  465. expect(entry.content).toEqual([
  466. {
  467. type: "text",
  468. text: parts.join(""),
  469. },
  470. ])
  471. expect(entry.time.completed).toEqual(time(parts.length + 3))
  472. }),
  473. { numRuns: 100 },
  474. )
  475. })
  476. test("replays prompt -> step -> text -> step.ended", () => {
  477. FastCheck.assert(
  478. FastCheck.property(word, texts, (body, parts) => {
  479. const next = run([
  480. SessionEvent.Prompt.create({ text: body, timestamp: time(0) }),
  481. SessionEvent.Step.Started.create({
  482. model: {
  483. id: "model",
  484. providerID: "provider",
  485. },
  486. timestamp: time(1),
  487. }),
  488. SessionEvent.Text.Started.create({ timestamp: time(2) }),
  489. ...parts.map((x, i) => SessionEvent.Text.Delta.create({ delta: x, timestamp: time(i + 3) })),
  490. SessionEvent.Step.Ended.create({
  491. reason: "stop",
  492. cost: 1,
  493. tokens: {
  494. input: 1,
  495. output: 2,
  496. reasoning: 3,
  497. cache: {
  498. read: 4,
  499. write: 5,
  500. },
  501. },
  502. timestamp: time(parts.length + 3),
  503. }),
  504. ])
  505. expect(next.entries).toHaveLength(2)
  506. expect(next.entries[0]?.type).toBe("user")
  507. expect(next.entries[1]?.type).toBe("assistant")
  508. if (next.entries[1]?.type !== "assistant") return
  509. expect(next.entries[1].content).toEqual([
  510. {
  511. type: "text",
  512. text: parts.join(""),
  513. },
  514. ])
  515. expect(next.entries[1].time.completed).toEqual(time(parts.length + 3))
  516. }),
  517. { numRuns: 50 },
  518. )
  519. })
  520. test("replays prompt -> step -> reasoning -> tool -> success -> step.ended", () => {
  521. FastCheck.assert(
  522. FastCheck.property(
  523. word,
  524. texts,
  525. text,
  526. dict,
  527. word,
  528. maybe(text),
  529. maybe(dict),
  530. maybe(files),
  531. (body, reason, end, input, title, output, metadata, attachments) => {
  532. const callID = "call"
  533. const next = run([
  534. SessionEvent.Prompt.create({ text: body, timestamp: time(0) }),
  535. SessionEvent.Step.Started.create({
  536. model: {
  537. id: "model",
  538. providerID: "provider",
  539. },
  540. timestamp: time(1),
  541. }),
  542. SessionEvent.Reasoning.Started.create({ timestamp: time(2) }),
  543. ...reason.map((x, i) => SessionEvent.Reasoning.Delta.create({ delta: x, timestamp: time(i + 3) })),
  544. SessionEvent.Reasoning.Ended.create({ text: end, timestamp: time(reason.length + 3) }),
  545. SessionEvent.Tool.Input.Started.create({ callID, name: "bash", timestamp: time(reason.length + 4) }),
  546. SessionEvent.Tool.Called.create({
  547. callID,
  548. tool: "bash",
  549. input,
  550. provider: { executed: true },
  551. timestamp: time(reason.length + 5),
  552. }),
  553. SessionEvent.Tool.Success.create({
  554. callID,
  555. title,
  556. output,
  557. metadata,
  558. attachments,
  559. provider: { executed: true },
  560. timestamp: time(reason.length + 6),
  561. }),
  562. SessionEvent.Step.Ended.create({
  563. reason: "stop",
  564. cost: 1,
  565. tokens: {
  566. input: 1,
  567. output: 2,
  568. reasoning: 3,
  569. cache: {
  570. read: 4,
  571. write: 5,
  572. },
  573. },
  574. timestamp: time(reason.length + 7),
  575. }),
  576. ])
  577. expect(next.entries.at(-1)?.type).toBe("assistant")
  578. const entry = next.entries.at(-1)
  579. if (entry?.type !== "assistant") return
  580. expect(entry.content).toHaveLength(2)
  581. expect(entry.content[0]).toEqual({
  582. type: "reasoning",
  583. text: end,
  584. })
  585. expect(entry.content[1]?.type).toBe("tool")
  586. if (entry.content[1]?.type !== "tool") return
  587. expect(entry.content[1].state.status).toBe("completed")
  588. expect(entry.time.completed).toEqual(time(reason.length + 7))
  589. },
  590. ),
  591. { numRuns: 50 },
  592. )
  593. })
  594. test("starting a new step completes the old assistant and appends a new active assistant", () => {
  595. const next = run(
  596. [
  597. SessionEvent.Step.Started.create({
  598. model: {
  599. id: "model",
  600. providerID: "provider",
  601. },
  602. timestamp: time(1),
  603. }),
  604. ],
  605. active(),
  606. )
  607. expect(next.entries).toHaveLength(2)
  608. expect(next.entries[0]?.type).toBe("assistant")
  609. expect(next.entries[1]?.type).toBe("assistant")
  610. if (next.entries[0]?.type !== "assistant" || next.entries[1]?.type !== "assistant") return
  611. expect(next.entries[0].time.completed).toEqual(time(1))
  612. expect(next.entries[1].time.created).toEqual(time(1))
  613. expect(next.entries[1].time.completed).toBeUndefined()
  614. })
  615. test("handles sequential tools independently", () => {
  616. FastCheck.assert(
  617. FastCheck.property(dict, dict, word, word, (a, b, title, error) => {
  618. const next = run(
  619. [
  620. SessionEvent.Tool.Input.Started.create({ callID: "a", name: "bash", timestamp: time(1) }),
  621. SessionEvent.Tool.Called.create({
  622. callID: "a",
  623. tool: "bash",
  624. input: a,
  625. provider: { executed: true },
  626. timestamp: time(2),
  627. }),
  628. SessionEvent.Tool.Success.create({
  629. callID: "a",
  630. title,
  631. output: "done",
  632. provider: { executed: true },
  633. timestamp: time(3),
  634. }),
  635. SessionEvent.Tool.Input.Started.create({ callID: "b", name: "grep", timestamp: time(4) }),
  636. SessionEvent.Tool.Called.create({
  637. callID: "b",
  638. tool: "bash",
  639. input: b,
  640. provider: { executed: true },
  641. timestamp: time(5),
  642. }),
  643. SessionEvent.Tool.Error.create({
  644. callID: "b",
  645. error,
  646. provider: { executed: true },
  647. timestamp: time(6),
  648. }),
  649. ],
  650. active(),
  651. )
  652. const first = tool(next, "a")
  653. const second = tool(next, "b")
  654. expect(first?.state.status).toBe("completed")
  655. if (first?.state.status !== "completed") return
  656. expect(first.state.input).toEqual(a)
  657. expect(first.state.output).toBe("done")
  658. expect(first.state.title).toBe(title)
  659. expect(second?.state.status).toBe("error")
  660. if (second?.state.status !== "error") return
  661. expect(second.state.input).toEqual(b)
  662. expect(second.state.error).toBe(error)
  663. }),
  664. { numRuns: 50 },
  665. )
  666. })
  667. test("routes tool events by callID when tool streams interleave", () => {
  668. FastCheck.assert(
  669. FastCheck.property(dict, dict, word, word, text, text, (a, b, titleA, titleB, deltaA, deltaB) => {
  670. const next = run(
  671. [
  672. SessionEvent.Tool.Input.Started.create({ callID: "a", name: "bash", timestamp: time(1) }),
  673. SessionEvent.Tool.Input.Started.create({ callID: "b", name: "grep", timestamp: time(2) }),
  674. SessionEvent.Tool.Input.Delta.create({ callID: "a", delta: deltaA, timestamp: time(3) }),
  675. SessionEvent.Tool.Input.Delta.create({ callID: "b", delta: deltaB, timestamp: time(4) }),
  676. SessionEvent.Tool.Called.create({
  677. callID: "a",
  678. tool: "bash",
  679. input: a,
  680. provider: { executed: true },
  681. timestamp: time(5),
  682. }),
  683. SessionEvent.Tool.Called.create({
  684. callID: "b",
  685. tool: "grep",
  686. input: b,
  687. provider: { executed: true },
  688. timestamp: time(6),
  689. }),
  690. SessionEvent.Tool.Success.create({
  691. callID: "a",
  692. title: titleA,
  693. output: "done-a",
  694. provider: { executed: true },
  695. timestamp: time(7),
  696. }),
  697. SessionEvent.Tool.Success.create({
  698. callID: "b",
  699. title: titleB,
  700. output: "done-b",
  701. provider: { executed: true },
  702. timestamp: time(8),
  703. }),
  704. ],
  705. active(),
  706. )
  707. const first = tool(next, "a")
  708. const second = tool(next, "b")
  709. expect(first?.state.status).toBe("completed")
  710. expect(second?.state.status).toBe("completed")
  711. if (first?.state.status !== "completed" || second?.state.status !== "completed") return
  712. expect(first.state.input).toEqual(a)
  713. expect(second.state.input).toEqual(b)
  714. expect(first.state.title).toBe(titleA)
  715. expect(second.state.title).toBe(titleB)
  716. }),
  717. { numRuns: 50 },
  718. )
  719. })
  720. test("records synthetic events", () => {
  721. FastCheck.assert(
  722. FastCheck.property(word, (body) => {
  723. const next = SessionEntryStepper.step(memoryState(), SessionEvent.Synthetic.create({ text: body, timestamp: time(1) }))
  724. expect(next.entries).toHaveLength(1)
  725. expect(next.entries[0]?.type).toBe("synthetic")
  726. if (next.entries[0]?.type !== "synthetic") return
  727. expect(next.entries[0].text).toBe(body)
  728. }),
  729. { numRuns: 50 },
  730. )
  731. })
  732. test("records compaction events", () => {
  733. FastCheck.assert(
  734. FastCheck.property(FastCheck.boolean(), maybe(FastCheck.boolean()), (auto, overflow) => {
  735. const next = SessionEntryStepper.step(
  736. memoryState(),
  737. SessionEvent.Compacted.create({ auto, overflow, timestamp: time(1) }),
  738. )
  739. expect(next.entries).toHaveLength(1)
  740. expect(next.entries[0]?.type).toBe("compaction")
  741. if (next.entries[0]?.type !== "compaction") return
  742. expect(next.entries[0].auto).toBe(auto)
  743. expect(next.entries[0].overflow).toBe(overflow)
  744. }),
  745. { numRuns: 50 },
  746. )
  747. })
  748. })
  749. })
  750. })