runtime.test.js 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206
  1. /** 工具注册表与 Agent Loop(Harness 运行时)测试。 */
  2. import { test } from 'node:test';
  3. import assert from 'node:assert/strict';
  4. import { ToolRegistry, defineTool } from '../src/harness/registry.js';
  5. import { EmbeddedHarness } from '../src/harness/runtime.js';
  6. import { ReplayLlmClient } from '../src/harness/llm.js';
  7. import { presetFromObject } from '../src/harness/preset.js';
  8. const preset = presetFromObject({
  9. name: 'test-agent',
  10. model: { provider: 'deepseek', name: 'deepseek-flash' },
  11. tools: ['echo'],
  12. systemPrompt: '测试用系统提示',
  13. limits: { maxSteps: 4 },
  14. });
  15. function makeRegistry(calls = []) {
  16. const registry = new ToolRegistry();
  17. registry.register(defineTool({
  18. name: 'echo',
  19. description: '回显并返回一行数据',
  20. parameters: { type: 'object', properties: { text: { type: 'string' } }, required: ['text'] },
  21. async execute(args, context) {
  22. calls.push({ args, callId: context.callId });
  23. return {
  24. content: { ok: true, echo: args.text },
  25. rows: [
  26. {
  27. values: { text: String(args.text ?? ''), amount: '123.45' },
  28. _trace: { object_id: 'obj_1', table: 't', row_index: 0, row_hash: 'sha256:row' },
  29. },
  30. ],
  31. objectIds: ['obj_1'],
  32. };
  33. },
  34. }));
  35. return registry;
  36. }
  37. function resolve() {
  38. return {
  39. provider: { id: 'deepseek', displayName: 'x', baseUrl: 'http://x', apiKey: 'k', api: 'openai-completions', models: ['deepseek-flash'], supportsTools: true },
  40. model: 'deepseek-flash',
  41. temperature: 0.1,
  42. maxOutputTokens: 1024,
  43. };
  44. }
  45. async function collect(stream) {
  46. const events = [];
  47. for await (const event of stream)
  48. events.push(event);
  49. return events;
  50. }
  51. test('defineTool 校验命名与 schema', () => {
  52. assert.throws(() => defineTool({ name: 'Bad Name', description: 'x', parameters: { type: 'object' }, execute: async () => ({ content: {} }) }));
  53. assert.throws(() => defineTool({ name: 'ok', description: '', parameters: { type: 'object' }, execute: async () => ({ content: {} }) }));
  54. assert.throws(() => defineTool({ name: 'ok', description: 'x', parameters: {}, execute: async () => ({ content: {} }) }));
  55. });
  56. test('注册表拒绝重复注册,只暴露允许的工具', () => {
  57. const registry = makeRegistry();
  58. assert.throws(() => registry.register(defineTool({ name: 'echo', description: 'x', parameters: { type: 'object' }, execute: async () => ({ content: {} }) })));
  59. const schemas = registry.schemas(['echo', 'not_exists']);
  60. assert.equal(schemas.length, 1);
  61. assert.equal(schemas[0]?.function.name, 'echo');
  62. });
  63. test('执行工具时 finalizeContent 注入 call_id 与行哈希', async () => {
  64. const registry = makeRegistry();
  65. const outcome = await registry.execute('echo', { text: 'hi' }, { callId: 'call_x', sessionId: 's', workspaceId: 'ws' });
  66. assert.equal(outcome.ok, true);
  67. const content = outcome.content;
  68. assert.equal(content.verification_context.call_id, 'call_x');
  69. assert.equal(content.verification_context.rows[0].row_hash, 'sha256:row');
  70. assert.equal(content.verification_context.row_count, 1);
  71. });
  72. test('工具抛错时返回结构化错误而不是崩溃', async () => {
  73. const registry = new ToolRegistry();
  74. registry.register(defineTool({ name: 'boom', description: 'x', parameters: { type: 'object' }, async execute() { throw new Error('炸了'); } }));
  75. const outcome = await registry.execute('boom', {}, { callId: 'c', sessionId: 's', workspaceId: 'w' });
  76. assert.equal(outcome.ok, false);
  77. assert.match(outcome.error ?? '', /炸了/);
  78. });
  79. test('未知工具返回可用工具列表', async () => {
  80. const registry = makeRegistry();
  81. const outcome = await registry.execute('nope', {}, { callId: 'c', sessionId: 's', workspaceId: 'w' });
  82. assert.equal(outcome.ok, false);
  83. assert.deepEqual(outcome.content.available, ['echo']);
  84. });
  85. test('Agent Loop:工具调用 → 结果回灌 → 产出最终答案', async () => {
  86. const calls = [];
  87. const registry = makeRegistry(calls);
  88. const ledgerRecords = [];
  89. const ledger = {
  90. record(input) {
  91. ledgerRecords.push(input);
  92. return { resultHash: 'sha256:x', rowCount: input.rows.length };
  93. },
  94. };
  95. const llm = new ReplayLlmClient([
  96. { text: '我先查一下。', toolCalls: [{ name: 'echo', arguments: { text: '你好' } }] },
  97. { text: '查询结果金额为 123.45 美元。' },
  98. ]);
  99. const harness = new EmbeddedHarness({ registry, llm, ledger, resolve });
  100. const events = await collect(harness.run({
  101. sessionId: 's1',
  102. workspaceId: 'ws1',
  103. messages: [{ role: 'user', content: '查一下' }],
  104. preset,
  105. }));
  106. const types = events.map((event) => event.type);
  107. assert.equal(types[0], 'started');
  108. assert.ok(types.includes('tool_call'));
  109. assert.ok(types.includes('tool_result'));
  110. const final = events.find((event) => event.type === 'final');
  111. assert.match(final.text, /123\.45/);
  112. assert.equal(final.toolCalls, 1);
  113. assert.equal(calls.length, 1);
  114. assert.equal(ledgerRecords.length, 1);
  115. assert.equal(ledgerRecords[0].callId, calls[0].callId);
  116. // 第二次请求必须带上工具结果(含溯源令牌)
  117. const secondCall = llm.seen[1];
  118. const toolMessage = secondCall.messages.find((message) => message.role === 'tool');
  119. assert.ok(toolMessage, '第二次调用缺少 tool 消息');
  120. assert.match(toolMessage.content ?? '', /verification_context/);
  121. assert.match(toolMessage.content ?? '', /call_replay_1_0/);
  122. });
  123. test('Agent Loop:多次工具调用的参数被正确累加解析', async () => {
  124. const registry = makeRegistry();
  125. const llm = new ReplayLlmClient([
  126. { toolCalls: [{ name: 'echo', arguments: { text: 'a' } }, { name: 'echo', arguments: { text: 'b' } }] },
  127. { text: '完成' },
  128. ]);
  129. const harness = new EmbeddedHarness({ registry, llm, resolve });
  130. const events = await collect(harness.run({ sessionId: 's', workspaceId: 'w', messages: [], preset }));
  131. const toolCalls = events.filter((event) => event.type === 'tool_call');
  132. assert.equal(toolCalls.length, 2);
  133. });
  134. test('Agent Loop:非法 JSON 参数不会让整个对话失败', async () => {
  135. const registry = makeRegistry();
  136. const llm = new ReplayLlmClient([{ toolCalls: [{ name: 'echo', arguments: { text: 'x' } }] }, { text: 'ok' }]);
  137. const harness = new EmbeddedHarness({ registry, llm, resolve });
  138. const events = await collect(harness.run({ sessionId: 's', workspaceId: 'w', messages: [], preset }));
  139. assert.ok(events.some((event) => event.type === 'final'));
  140. });
  141. test('Agent Loop:达到最大步数后强制给出结论', async () => {
  142. const registry = makeRegistry();
  143. const steps = Array.from({ length: preset.limits.maxSteps }, () => ({
  144. toolCalls: [{ name: 'echo', arguments: { text: 'loop' } }],
  145. }));
  146. const llm = new ReplayLlmClient([...steps, { text: '被迫总结:没有更多数据。' }]);
  147. const harness = new EmbeddedHarness({ registry, llm, resolve });
  148. const events = await collect(harness.run({ sessionId: 's', workspaceId: 'w', messages: [], preset }));
  149. const final = events.find((event) => event.type === 'final');
  150. assert.equal(final.steps, preset.limits.maxSteps);
  151. assert.match(final.text, /被迫总结/);
  152. // 最后一次调用禁用了工具
  153. assert.equal(llm.seen.at(-1)?.tools.length, 0);
  154. });
  155. test('collectScalars 只收标量,且跳过 rows / 令牌', async () => {
  156. const { collectScalars } = await import('../src/harness/runtime.js');
  157. const scalars = collectScalars({
  158. object_id: 'obj_1',
  159. total_matching_rows: 20,
  160. exhausted: true,
  161. stats: { included: 1057, seconds: 11.8 },
  162. rows: [{ a: 1 }],
  163. verification_context: { call_id: 'x', row_count: 1 },
  164. });
  165. assert.equal(scalars['total_matching_rows'], 20);
  166. assert.equal(scalars['stats.included'], 1057);
  167. assert.equal(scalars['stats.seconds'], 11.8);
  168. assert.equal(scalars['exhausted'], undefined);
  169. assert.equal(scalars['rows'], undefined);
  170. assert.equal(scalars['verification_context'], undefined);
  171. // 数字字符串也收(Python/JS 之间数量可能以字符串传递)
  172. assert.equal(collectScalars({ total: '18695479' })['total'], '18695479');
  173. });
  174. test('Agent Loop:工具返回的标量也会写进台账(供核验反查)', async () => {
  175. const registry = new ToolRegistry();
  176. registry.register(defineTool({
  177. name: 'stat',
  178. description: '返回带标量的结果',
  179. parameters: { type: 'object' },
  180. async execute() {
  181. return { content: { total_matching_rows: 42, stats: { included: 1057 } }, rows: [], objectIds: ['obj_1'] };
  182. },
  183. }));
  184. const records = [];
  185. const ledger = { record: (input) => records.push(input) };
  186. const llm = new ReplayLlmClient([{ toolCalls: [{ name: 'stat', arguments: {} }] }, { text: '共 42 行' }]);
  187. const harness = new EmbeddedHarness({ registry, llm, ledger, resolve });
  188. await collect(harness.run({ sessionId: 's', workspaceId: 'w', messages: [], preset }));
  189. assert.equal(records.length, 1);
  190. assert.equal(records[0].extraValues.total_matching_rows, 42);
  191. assert.equal(records[0].extraValues['stats.included'], 1057);
  192. });
  193. test('Agent Loop:LLM 报错时产出 error 事件', async () => {
  194. const registry = makeRegistry();
  195. const broken = {
  196. name: 'broken',
  197. async *stream() {
  198. throw new Error('模型连接失败');
  199. },
  200. };
  201. const harness = new EmbeddedHarness({ registry, llm: broken, resolve });
  202. const events = await collect(harness.run({ sessionId: 's', workspaceId: 'w', messages: [], preset }));
  203. const error = events.find((event) => event.type === 'error');
  204. assert.match(error.message, /模型连接失败/);
  205. });
  206. //# sourceMappingURL=runtime.test.js.map