runtime.test.ts 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234
  1. /** 工具注册表与 Agent Loop(Harness 运行时)测试。 */
  2. import { test } from 'node:test';
  3. import assert from 'node:assert/strict';
  4. import { ToolRegistry, defineTool } from '../src/harness/registry.js';
  5. import { EmbeddedHarness } from '../src/harness/runtime.js';
  6. import { ReplayLlmClient } from '../src/harness/llm.js';
  7. import { presetFromObject } from '../src/harness/preset.js';
  8. import type { HarnessEvent } from '../src/harness/types.js';
  9. const preset = presetFromObject({
  10. name: 'test-agent',
  11. model: { provider: 'deepseek', name: 'deepseek-flash' },
  12. tools: ['echo'],
  13. systemPrompt: '测试用系统提示',
  14. limits: { maxSteps: 4 },
  15. });
  16. function makeRegistry(calls: Array<Record<string, unknown>> = []): ToolRegistry {
  17. const registry = new ToolRegistry();
  18. registry.register(
  19. defineTool({
  20. name: 'echo',
  21. description: '回显并返回一行数据',
  22. parameters: { type: 'object', properties: { text: { type: 'string' } }, required: ['text'] },
  23. async execute(args, context) {
  24. calls.push({ args, callId: context.callId });
  25. return {
  26. content: { ok: true, echo: args.text },
  27. rows: [
  28. {
  29. values: { text: String(args.text ?? ''), amount: '123.45' },
  30. _trace: { object_id: 'obj_1', table: 't', row_index: 0, row_hash: 'sha256:row' },
  31. },
  32. ],
  33. objectIds: ['obj_1'],
  34. };
  35. },
  36. }),
  37. );
  38. return registry;
  39. }
  40. function resolve() {
  41. return {
  42. provider: { id: 'deepseek', displayName: 'x', baseUrl: 'http://x', apiKey: 'k', api: 'openai-completions' as const, models: ['deepseek-flash'], supportsTools: true },
  43. model: 'deepseek-flash',
  44. temperature: 0.1,
  45. maxOutputTokens: 1024,
  46. };
  47. }
  48. async function collect(stream: AsyncGenerator<HarnessEvent>): Promise<HarnessEvent[]> {
  49. const events: HarnessEvent[] = [];
  50. for await (const event of stream) events.push(event);
  51. return events;
  52. }
  53. test('defineTool 校验命名与 schema', () => {
  54. assert.throws(() => defineTool({ name: 'Bad Name', description: 'x', parameters: { type: 'object' }, execute: async () => ({ content: {} }) }));
  55. assert.throws(() => defineTool({ name: 'ok', description: '', parameters: { type: 'object' }, execute: async () => ({ content: {} }) }));
  56. assert.throws(() => defineTool({ name: 'ok', description: 'x', parameters: {}, execute: async () => ({ content: {} }) }));
  57. });
  58. test('注册表拒绝重复注册,只暴露允许的工具', () => {
  59. const registry = makeRegistry();
  60. assert.throws(() =>
  61. registry.register(defineTool({ name: 'echo', description: 'x', parameters: { type: 'object' }, execute: async () => ({ content: {} }) })),
  62. );
  63. const schemas = registry.schemas(['echo', 'not_exists']);
  64. assert.equal(schemas.length, 1);
  65. assert.equal(schemas[0]?.function.name, 'echo');
  66. });
  67. test('执行工具时 finalizeContent 注入 call_id 与行哈希', async () => {
  68. const registry = makeRegistry();
  69. const outcome = await registry.execute('echo', { text: 'hi' }, { callId: 'call_x', sessionId: 's', workspaceId: 'ws' });
  70. assert.equal(outcome.ok, true);
  71. const content = outcome.content as Record<string, any>;
  72. assert.equal(content.verification_context.call_id, 'call_x');
  73. assert.equal(content.verification_context.rows[0].row_hash, 'sha256:row');
  74. assert.equal(content.verification_context.row_count, 1);
  75. });
  76. test('工具抛错时返回结构化错误而不是崩溃', async () => {
  77. const registry = new ToolRegistry();
  78. registry.register(
  79. defineTool({ name: 'boom', description: 'x', parameters: { type: 'object' }, async execute() { throw new Error('炸了'); } }),
  80. );
  81. const outcome = await registry.execute('boom', {}, { callId: 'c', sessionId: 's', workspaceId: 'w' });
  82. assert.equal(outcome.ok, false);
  83. assert.match(outcome.error ?? '', /炸了/);
  84. });
  85. test('未知工具返回可用工具列表', async () => {
  86. const registry = makeRegistry();
  87. const outcome = await registry.execute('nope', {}, { callId: 'c', sessionId: 's', workspaceId: 'w' });
  88. assert.equal(outcome.ok, false);
  89. assert.deepEqual((outcome.content as any).available, ['echo']);
  90. });
  91. test('Agent Loop:工具调用 → 结果回灌 → 产出最终答案', async () => {
  92. const calls: Array<Record<string, unknown>> = [];
  93. const registry = makeRegistry(calls);
  94. const ledgerRecords: any[] = [];
  95. const ledger = {
  96. record(input: any) {
  97. ledgerRecords.push(input);
  98. return { resultHash: 'sha256:x', rowCount: input.rows.length };
  99. },
  100. } as any;
  101. const llm = new ReplayLlmClient([
  102. { text: '我先查一下。', toolCalls: [{ name: 'echo', arguments: { text: '你好' } }] },
  103. { text: '查询结果金额为 123.45 美元。' },
  104. ]);
  105. const harness = new EmbeddedHarness({ registry, llm, ledger, resolve });
  106. const events = await collect(
  107. harness.run({
  108. sessionId: 's1',
  109. workspaceId: 'ws1',
  110. messages: [{ role: 'user', content: '查一下' }],
  111. preset,
  112. }),
  113. );
  114. const types = events.map((event) => event.type);
  115. assert.equal(types[0], 'started');
  116. assert.ok(types.includes('tool_call'));
  117. assert.ok(types.includes('tool_result'));
  118. const final = events.find((event) => event.type === 'final') as Extract<HarnessEvent, { type: 'final' }>;
  119. assert.match(final.text, /123\.45/);
  120. assert.equal(final.toolCalls, 1);
  121. assert.equal(calls.length, 1);
  122. assert.equal(ledgerRecords.length, 1);
  123. assert.equal(ledgerRecords[0].callId, calls[0].callId);
  124. // 第二次请求必须带上工具结果(含溯源令牌)
  125. const secondCall = llm.seen[1];
  126. const toolMessage = secondCall.messages.find((message) => message.role === 'tool');
  127. assert.ok(toolMessage, '第二次调用缺少 tool 消息');
  128. assert.match(toolMessage.content ?? '', /verification_context/);
  129. assert.match(toolMessage.content ?? '', /call_replay_1_0/);
  130. });
  131. test('Agent Loop:多次工具调用的参数被正确累加解析', async () => {
  132. const registry = makeRegistry();
  133. const llm = new ReplayLlmClient([
  134. { toolCalls: [{ name: 'echo', arguments: { text: 'a' } }, { name: 'echo', arguments: { text: 'b' } }] },
  135. { text: '完成' },
  136. ]);
  137. const harness = new EmbeddedHarness({ registry, llm, resolve });
  138. const events = await collect(harness.run({ sessionId: 's', workspaceId: 'w', messages: [], preset }));
  139. const toolCalls = events.filter((event) => event.type === 'tool_call');
  140. assert.equal(toolCalls.length, 2);
  141. });
  142. test('Agent Loop:非法 JSON 参数不会让整个对话失败', async () => {
  143. const registry = makeRegistry();
  144. const llm = new ReplayLlmClient([{ toolCalls: [{ name: 'echo', arguments: { text: 'x' } }] }, { text: 'ok' }]);
  145. const harness = new EmbeddedHarness({ registry, llm, resolve });
  146. const events = await collect(harness.run({ sessionId: 's', workspaceId: 'w', messages: [], preset }));
  147. assert.ok(events.some((event) => event.type === 'final'));
  148. });
  149. test('Agent Loop:达到最大步数后强制给出结论', async () => {
  150. const registry = makeRegistry();
  151. const steps = Array.from({ length: preset.limits.maxSteps }, () => ({
  152. toolCalls: [{ name: 'echo', arguments: { text: 'loop' } }],
  153. }));
  154. const llm = new ReplayLlmClient([...steps, { text: '被迫总结:没有更多数据。' }]);
  155. const harness = new EmbeddedHarness({ registry, llm, resolve });
  156. const events = await collect(harness.run({ sessionId: 's', workspaceId: 'w', messages: [], preset }));
  157. const final = events.find((event) => event.type === 'final') as Extract<HarnessEvent, { type: 'final' }>;
  158. assert.equal(final.steps, preset.limits.maxSteps);
  159. assert.match(final.text, /被迫总结/);
  160. // 最后一次调用禁用了工具
  161. assert.equal(llm.seen.at(-1)?.tools.length, 0);
  162. });
  163. test('collectScalars 只收标量,且跳过 rows / 令牌', async () => {
  164. const { collectScalars } = await import('../src/harness/runtime.js');
  165. const scalars = collectScalars({
  166. object_id: 'obj_1',
  167. total_matching_rows: 20,
  168. exhausted: true,
  169. stats: { included: 1057, seconds: 11.8 },
  170. rows: [{ a: 1 }],
  171. verification_context: { call_id: 'x', row_count: 1 },
  172. });
  173. assert.equal(scalars['total_matching_rows'], 20);
  174. assert.equal(scalars['stats.included'], 1057);
  175. assert.equal(scalars['stats.seconds'], 11.8);
  176. assert.equal(scalars['exhausted'], undefined);
  177. assert.equal(scalars['rows'], undefined);
  178. assert.equal(scalars['verification_context'], undefined);
  179. // 数字字符串也收(Python/JS 之间数量可能以字符串传递)
  180. assert.equal(collectScalars({ total: '18695479' })['total'], '18695479');
  181. });
  182. test('Agent Loop:工具返回的标量也会写进台账(供核验反查)', async () => {
  183. const registry = new ToolRegistry();
  184. registry.register(
  185. defineTool({
  186. name: 'stat',
  187. description: '返回带标量的结果',
  188. parameters: { type: 'object' },
  189. async execute() {
  190. return { content: { total_matching_rows: 42, stats: { included: 1057 } }, rows: [], objectIds: ['obj_1'] };
  191. },
  192. }),
  193. );
  194. const records: any[] = [];
  195. const ledger = { record: (input: any) => records.push(input) } as any;
  196. const llm = new ReplayLlmClient([{ toolCalls: [{ name: 'stat', arguments: {} }] }, { text: '共 42 行' }]);
  197. const harness = new EmbeddedHarness({ registry, llm, ledger, resolve });
  198. await collect(harness.run({ sessionId: 's', workspaceId: 'w', messages: [], preset }));
  199. assert.equal(records.length, 1);
  200. assert.equal(records[0].extraValues.total_matching_rows, 42);
  201. assert.equal(records[0].extraValues['stats.included'], 1057);
  202. });
  203. test('Agent Loop:LLM 报错时产出 error 事件', async () => {
  204. const registry = makeRegistry();
  205. const broken: any = {
  206. name: 'broken',
  207. async *stream() {
  208. throw new Error('模型连接失败');
  209. },
  210. };
  211. const harness = new EmbeddedHarness({ registry, llm: broken, resolve });
  212. const events = await collect(harness.run({ sessionId: 's', workspaceId: 'w', messages: [], preset }));
  213. const error = events.find((event) => event.type === 'error') as Extract<HarnessEvent, { type: 'error' }>;
  214. assert.match(error.message, /模型连接失败/);
  215. });