usage.test.ts 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314
  1. import { describe, expect, test } from "bun:test"
  2. import type { SessionNotification } from "@agentclientprotocol/sdk"
  3. import { UsageService } from "@/acp/usage"
  4. import { ModelID, ProviderID } from "@/provider/schema"
  5. import { Provider } from "@/provider/provider"
  6. import { Effect, Layer } from "effect"
  7. import { it } from "../lib/effect"
  8. const assistant = (
  9. input: Partial<UsageService.AssistantMessage> & Pick<UsageService.AssistantMessage, "cost">,
  10. ): UsageService.SessionMessage => ({
  11. info: {
  12. role: "assistant",
  13. providerID: "anthropic",
  14. modelID: "claude-sonnet",
  15. tokens: {
  16. input: 10,
  17. output: 20,
  18. reasoning: 0,
  19. cache: { read: 0, write: 0 },
  20. },
  21. ...input,
  22. },
  23. })
  24. const user = (): UsageService.SessionMessage => ({
  25. info: { role: "user" },
  26. })
  27. const assistantWithoutProvider = (): UsageService.SessionMessage => ({
  28. info: {
  29. role: "assistant",
  30. modelID: "claude-sonnet",
  31. cost: 1,
  32. tokens: {
  33. input: 10,
  34. output: 20,
  35. reasoning: 0,
  36. cache: { read: 0, write: 0 },
  37. },
  38. },
  39. })
  40. const model = (providerID: ProviderID, modelID: ModelID, context: number): Provider.Model => ({
  41. id: modelID,
  42. providerID,
  43. api: {
  44. id: modelID,
  45. url: "https://example.com",
  46. npm: "@ai-sdk/openai-compatible",
  47. },
  48. name: modelID,
  49. family: "test",
  50. capabilities: {
  51. temperature: true,
  52. reasoning: false,
  53. attachment: false,
  54. toolcall: true,
  55. input: { text: true, audio: false, image: false, video: false, pdf: false },
  56. output: { text: true, audio: false, image: false, video: false, pdf: false },
  57. interleaved: false,
  58. },
  59. cost: {
  60. input: 0,
  61. output: 0,
  62. cache: { read: 0, write: 0 },
  63. },
  64. limit: {
  65. context,
  66. output: 4096,
  67. },
  68. status: "active",
  69. options: {},
  70. headers: {},
  71. release_date: "2026-01-01",
  72. })
  73. const providers = (context = 128_000): Record<ProviderID, Provider.Info> => {
  74. const providerID = ProviderID.make("anthropic")
  75. const modelID = ModelID.make("claude-sonnet")
  76. return {
  77. [providerID]: {
  78. id: providerID,
  79. name: "Anthropic",
  80. source: "config",
  81. env: [],
  82. options: {},
  83. models: {
  84. [modelID]: model(providerID, modelID, context),
  85. },
  86. },
  87. }
  88. }
  89. const fakeLayer = (input: {
  90. readonly messages?: Effect.Effect<readonly UsageService.SessionMessage[], unknown>
  91. readonly providers?: (directory: string) => Effect.Effect<Record<ProviderID, Provider.Info>, unknown>
  92. }) =>
  93. UsageService.layer.pipe(
  94. Layer.provide(
  95. Layer.mergeAll(
  96. Layer.succeed(
  97. UsageService.MessageLoader,
  98. UsageService.MessageLoader.of({
  99. messages: () => input.messages ?? Effect.succeed([]),
  100. }),
  101. ),
  102. Layer.succeed(
  103. UsageService.ContextLimitLoader,
  104. UsageService.ContextLimitLoader.of({
  105. providers: input.providers ?? (() => Effect.succeed(providers())),
  106. }),
  107. ),
  108. ),
  109. ),
  110. )
  111. const connection = (updates: SessionNotification[]) => ({
  112. sessionUpdate(params: SessionNotification) {
  113. updates.push(params)
  114. return Promise.resolve()
  115. },
  116. })
  117. describe("acp usage", () => {
  118. test("builds ACP Usage from assistant token shape", () => {
  119. expect(
  120. UsageService.buildUsage({
  121. cost: 0.02,
  122. tokens: {
  123. input: 100,
  124. output: 40,
  125. reasoning: 7,
  126. cache: { read: 11, write: 13 },
  127. },
  128. }),
  129. ).toEqual({
  130. inputTokens: 100,
  131. outputTokens: 40,
  132. thoughtTokens: 7,
  133. cachedReadTokens: 11,
  134. cachedWriteTokens: 13,
  135. totalTokens: 171,
  136. })
  137. })
  138. test("omits optional token fields when they are zero", () => {
  139. expect(
  140. UsageService.buildUsage({
  141. cost: 0,
  142. tokens: {
  143. input: 3,
  144. output: 4,
  145. reasoning: 0,
  146. cache: { read: 0, write: 0 },
  147. },
  148. }),
  149. ).toEqual({
  150. inputTokens: 3,
  151. outputTokens: 4,
  152. totalTokens: 7,
  153. })
  154. })
  155. test("finds the latest assistant message", () => {
  156. expect(
  157. UsageService.latestAssistantMessage([assistant({ cost: 1, modelID: "older" }), user(), assistant({ cost: 2 })]),
  158. ).toMatchObject({ cost: 2 })
  159. })
  160. test("calculates total session cost from assistant messages", () => {
  161. expect(UsageService.totalSessionCost([assistant({ cost: 1.25 }), user(), assistant({ cost: 2.5 })])).toBe(3.75)
  162. })
  163. it.effect("loads context limits from providers and caches by directory/provider/model", () => {
  164. const calls: string[] = []
  165. return Effect.gen(function* () {
  166. const usage = yield* UsageService.Service
  167. const first = yield* usage.contextLimit({
  168. directory: "/workspace",
  169. providerID: ProviderID.make("anthropic"),
  170. modelID: ModelID.make("claude-sonnet"),
  171. })
  172. const second = yield* usage.contextLimit({
  173. directory: "/workspace",
  174. providerID: ProviderID.make("anthropic"),
  175. modelID: ModelID.make("claude-sonnet"),
  176. })
  177. expect(first).toBe(200_000)
  178. expect(second).toBe(200_000)
  179. expect(calls).toEqual(["/workspace"])
  180. }).pipe(
  181. Effect.provide(
  182. fakeLayer({
  183. providers: (directory) =>
  184. Effect.sync(() => {
  185. calls.push(directory)
  186. return providers(200_000)
  187. }),
  188. }),
  189. ),
  190. )
  191. })
  192. it.effect("sends ACP usage_update with context size and cumulative assistant cost", () => {
  193. const updates: SessionNotification[] = []
  194. return Effect.gen(function* () {
  195. const usage = yield* UsageService.Service
  196. yield* usage.sendUpdate({
  197. connection: connection(updates),
  198. sessionID: "ses_1",
  199. directory: "/workspace",
  200. })
  201. expect(updates).toEqual([
  202. {
  203. sessionId: "ses_1",
  204. update: {
  205. sessionUpdate: "usage_update",
  206. used: 15,
  207. size: 128_000,
  208. cost: { amount: 3, currency: "USD" },
  209. },
  210. },
  211. ])
  212. }).pipe(
  213. Effect.provide(
  214. fakeLayer({
  215. messages: Effect.succeed([
  216. assistant({ cost: 1 }),
  217. assistant({
  218. cost: 2,
  219. tokens: {
  220. input: 10,
  221. output: 20,
  222. reasoning: 0,
  223. cache: { read: 5, write: 0 },
  224. },
  225. }),
  226. ]),
  227. }),
  228. ),
  229. )
  230. })
  231. it.effect("skips usage update when messages cannot be fetched", () => {
  232. const updates: SessionNotification[] = []
  233. return Effect.gen(function* () {
  234. const usage = yield* UsageService.Service
  235. yield* usage.sendUpdate({
  236. connection: connection(updates),
  237. sessionID: "ses_1",
  238. directory: "/workspace",
  239. })
  240. expect(updates).toEqual([])
  241. }).pipe(Effect.provide(fakeLayer({ messages: Effect.fail(new Error("boom")) })))
  242. })
  243. it.effect("skips usage update when no assistant message exists", () => {
  244. const updates: SessionNotification[] = []
  245. return Effect.gen(function* () {
  246. const usage = yield* UsageService.Service
  247. yield* usage.sendUpdate({
  248. connection: connection(updates),
  249. sessionID: "ses_1",
  250. directory: "/workspace",
  251. })
  252. expect(updates).toEqual([])
  253. }).pipe(Effect.provide(fakeLayer({ messages: Effect.succeed([user()]) })))
  254. })
  255. it.effect("skips usage update when assistant message has no provider or model", () => {
  256. const updates: SessionNotification[] = []
  257. return Effect.gen(function* () {
  258. const usage = yield* UsageService.Service
  259. yield* usage.sendUpdate({
  260. connection: connection(updates),
  261. sessionID: "ses_1",
  262. directory: "/workspace",
  263. })
  264. expect(updates).toEqual([])
  265. }).pipe(
  266. Effect.provide(
  267. fakeLayer({
  268. messages: Effect.succeed([assistantWithoutProvider()]),
  269. }),
  270. ),
  271. )
  272. })
  273. it.effect("skips usage update when context size is unknown", () => {
  274. const updates: SessionNotification[] = []
  275. return Effect.gen(function* () {
  276. const usage = yield* UsageService.Service
  277. yield* usage.sendUpdate({
  278. connection: connection(updates),
  279. sessionID: "ses_1",
  280. directory: "/workspace",
  281. })
  282. expect(updates).toEqual([])
  283. }).pipe(
  284. Effect.provide(
  285. fakeLayer({
  286. messages: Effect.succeed([assistant({ cost: 1, providerID: "missing" })]),
  287. }),
  288. ),
  289. )
  290. })
  291. })