ledger.py 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215
  1. """观测台账:每次工具调用一条不可篡改记录,并索引所有返回过的字段值。
  2. 这是“数据真实性核验”的后端事实来源:
  3. * ``calls`` —— 调用级记录,``prev_hash`` + ``hash`` 构成哈希链;
  4. * ``value_index`` —— 调用返回值里的每个字段值一行,供核验层按
  5. ``(call_id, row_index, field, value)`` 精确反查;
  6. * ``ledger.jsonl`` —— 追加式落盘副本,便于离线审计或被外部系统消费。
  7. 没有登记在台账里的数字,一律视为“无来源”。
  8. """
  9. from __future__ import annotations
  10. import hashlib
  11. import json
  12. import sqlite3
  13. from pathlib import Path
  14. from .config import settings
  15. from .db import database
  16. def _sha256(text: str) -> str:
  17. return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
  18. def _canonical(payload: object) -> str:
  19. return json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str)
  20. def _to_float(value: object) -> float | None:
  21. try:
  22. if isinstance(value, bool):
  23. return None
  24. return float(str(value).replace(",", ""))
  25. except (TypeError, ValueError):
  26. return None
  27. class Ledger:
  28. """调用台账。"""
  29. def __init__(self, path: str | Path | None = None) -> None:
  30. self.path = Path(path or settings.ledger_path)
  31. self.enabled = settings.ledger_jsonl
  32. if self.enabled:
  33. self.path.parent.mkdir(parents=True, exist_ok=True)
  34. # ------------------------------------------------------------------ writing
  35. def record(
  36. self,
  37. call_id: str,
  38. tool: str,
  39. args: dict,
  40. result: dict,
  41. *,
  42. object_ids: list[str] | None = None,
  43. session_id: str = "",
  44. rows: list[dict] | None = None,
  45. verdict_hint: str = "",
  46. ) -> dict:
  47. """登记一次调用。``rows`` 用于建立字段级索引。"""
  48. with database.connect() as connection:
  49. previous = connection.execute("SELECT hash FROM calls ORDER BY rowid DESC LIMIT 1").fetchone()
  50. prev_hash = previous["hash"] if previous else ""
  51. result_hash = _sha256(_canonical(result))
  52. args_hash = _sha256(_canonical(args))
  53. entry_hash = _sha256(f"{prev_hash}|{call_id}|{tool}|{args_hash}|{result_hash}|{session_id}")
  54. with database.transaction() as connection:
  55. connection.execute(
  56. "INSERT OR REPLACE INTO calls(call_id,tool,args_hash,args_json,result_hash,result_json,object_ids,"
  57. "verdict_hint,prev_hash,hash,session_id,created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,datetime('now','localtime'))",
  58. (
  59. call_id,
  60. tool,
  61. args_hash,
  62. _canonical(args),
  63. result_hash,
  64. _canonical(result)[:200000],
  65. json.dumps(object_ids or [], ensure_ascii=False),
  66. verdict_hint,
  67. prev_hash,
  68. entry_hash,
  69. session_id,
  70. ),
  71. )
  72. if rows:
  73. self._index_values(connection, call_id, session_id, rows)
  74. entry = {
  75. "call_id": call_id,
  76. "tool": tool,
  77. "args_hash": args_hash,
  78. "result_hash": result_hash,
  79. "object_ids": object_ids or [],
  80. "session_id": session_id,
  81. "prev_hash": prev_hash,
  82. "hash": entry_hash,
  83. }
  84. if self.enabled:
  85. with self.path.open("a", encoding="utf-8") as handle:
  86. handle.write(json.dumps(entry, ensure_ascii=False) + "\n")
  87. return entry
  88. @staticmethod
  89. def _index_values(connection: sqlite3.Connection, call_id: str, session_id: str, rows: list[dict]) -> None:
  90. entries: list[tuple] = []
  91. for row in rows:
  92. trace = row.get("_trace") or {}
  93. values = row.get("values") or {}
  94. for field, value in values.items():
  95. if value is None:
  96. continue
  97. entries.append(
  98. (
  99. call_id,
  100. session_id,
  101. str(trace.get("object_id") or ""),
  102. str(trace.get("table") or ""),
  103. int(trace.get("row_index", -1)),
  104. str(field),
  105. str(value),
  106. _to_float(value),
  107. )
  108. )
  109. if entries:
  110. connection.executemany(
  111. "INSERT INTO value_index(call_id,session_id,object_id,table_name,row_index,field,value_text,value_num) "
  112. "VALUES(?,?,?,?,?,?,?,?)",
  113. entries,
  114. )
  115. # ------------------------------------------------------------------ reading
  116. def get(self, call_id: str) -> dict | None:
  117. with database.connect() as connection:
  118. row = connection.execute("SELECT * FROM calls WHERE call_id=?", (call_id,)).fetchone()
  119. if not row:
  120. return None
  121. item = dict(row)
  122. item["args"] = json.loads(item.pop("args_json") or "{}")
  123. item["result"] = json.loads(item.pop("result_json") or "{}")
  124. item["object_ids"] = json.loads(item["object_ids"] or "[]")
  125. return item
  126. def session_calls(self, session_id: str) -> list[dict]:
  127. with database.connect() as connection:
  128. rows = connection.execute(
  129. "SELECT call_id,tool,args_hash,result_hash,object_ids,verdict_hint,created_at,hash "
  130. "FROM calls WHERE session_id=? ORDER BY rowid",
  131. (session_id,),
  132. ).fetchall()
  133. return [dict(row) for row in rows]
  134. def find_value(self, value: str, session_id: str = "", tolerance: float = 1e-9) -> list[dict]:
  135. """在一个会话的台账里反查字段值,返回命中明细。"""
  136. number = _to_float(value)
  137. with database.connect() as connection:
  138. if number is not None:
  139. rows = connection.execute(
  140. "SELECT call_id,object_id,table_name,row_index,field,value_text,value_num FROM value_index "
  141. "WHERE session_id=? AND value_num IS NOT NULL AND ABS(value_num - ?) <= ? "
  142. "ORDER BY rowid DESC LIMIT 50",
  143. (session_id, number, tolerance * max(1.0, abs(number))),
  144. ).fetchall()
  145. if rows:
  146. return [dict(row) for row in rows]
  147. rows = connection.execute(
  148. "SELECT call_id,object_id,table_name,row_index,field,value_text,value_num FROM value_index "
  149. "WHERE session_id=? AND value_text=? ORDER BY rowid DESC LIMIT 50",
  150. (session_id, str(value)),
  151. ).fetchall()
  152. return [dict(row) for row in rows]
  153. def find_calculation(self, calculation_id: str) -> dict | None:
  154. item = self.get(calculation_id)
  155. if not item or item["tool"] != "calculator":
  156. return None
  157. return item
  158. def chain_ok(self, limit: int = 200) -> bool:
  159. """校验哈希链(运维自检用)。"""
  160. with database.connect() as connection:
  161. rows = connection.execute(
  162. "SELECT call_id,tool,args_hash,result_hash,session_id,prev_hash,hash FROM calls ORDER BY rowid LIMIT ?",
  163. (limit,),
  164. ).fetchall()
  165. previous = ""
  166. for row in rows:
  167. expected = _sha256(
  168. f"{row['prev_hash']}|{row['call_id']}|{row['tool']}|{row['args_hash']}|{row['result_hash']}|{row['session_id']}"
  169. )
  170. if row["prev_hash"] != previous or row["hash"] != expected:
  171. return False
  172. previous = row["hash"]
  173. return True
  174. def stats(self) -> dict:
  175. with database.connect() as connection:
  176. calls = connection.execute("SELECT COUNT(*) FROM calls").fetchone()[0]
  177. values = connection.execute("SELECT COUNT(*) FROM value_index").fetchone()[0]
  178. last = connection.execute("SELECT call_id,tool,created_at FROM calls ORDER BY rowid DESC LIMIT 1").fetchone()
  179. return {
  180. "calls": calls,
  181. "indexed_values": values,
  182. "last_call": dict(last) if last else None,
  183. "chain_ok": self.chain_ok(limit=min(calls, 500)),
  184. }
  185. ledger = Ledger()