test_format_matrix.py 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191
  1. """Generated format matrix, accounting invariants and precision regressions."""
  2. import csv
  3. import sqlite3
  4. from decimal import Decimal
  5. from pathlib import Path
  6. import pytest
  7. from toolsvc.customs import CustomsOptions, CustomsProcessRequest, load_processor
  8. from toolsvc.customs.engine.numbers import number
  9. from toolsvc.customs.engine.rules import Rules
  10. from toolsvc.customs.engine.storage import Store, ingest
  11. from toolsvc.objects import _iter_rows, store
  12. from toolsvc.source import iter_source
  13. HEAD = ['I_E_FLAG', 'HS_CODE', 'YEAR', 'WEEK', 'MONORSUM_USD', 'COUNT', 'G_MODEL']
  14. ROW = ['I', '8532241000', '2025', '1', '1234.50', '10', '品牌:MURATA;型号:TEST-1']
  15. RULE_DIR = Path(__file__).resolve().parents[1] / 'toolsvc/customs/rules'
  16. @pytest.fixture(scope='module')
  17. def rules():
  18. return Rules(RULE_DIR / '数据映射规则.xlsx', RULE_DIR / '进出口数据字段需求20260819.xlsx')
  19. def config(path, **extra):
  20. return dict(input_csv=str(path), start_year=2025, end_year=2026,
  21. directions=['I'], cutoff_date='2026-12-31', **extra)
  22. @pytest.mark.parametrize('encoding', ['utf-8', 'utf-8-sig', 'gb18030', 'utf-16'])
  23. @pytest.mark.parametrize('delimiter', [',', '\t', ';', '|'])
  24. def test_upload_and_ingest_same_formats(tmp_path, rules, encoding, delimiter):
  25. path = tmp_path / 'input.csv'
  26. with path.open('w', encoding=encoding, newline='') as f:
  27. w = csv.writer(f, delimiter=delimiter)
  28. w.writerow(HEAD)
  29. w.writerow(ROW)
  30. headers, iterator = _iter_rows(path)
  31. assert headers == HEAD
  32. assert list(iterator) == [ROW]
  33. db, stats = ingest(config(path), rules, tmp_path)
  34. try:
  35. assert stats['included'] == 1
  36. assert Store(db).total({})['amount'] == Decimal('1234.50')
  37. finally:
  38. db.close()
  39. @pytest.mark.parametrize('raw,expected', [
  40. ('1,234.50', '1234.50'), ('1,234.50', '1234.50'),
  41. ('1 234.50', '1234.50'), ('$1,234.50', '1234.50'),
  42. ('1234.50美元', '1234.50'), ('(1,234.50)', '-1234.50'),
  43. ('−100', '-100'), ('1.2345e3', '1234.5'),
  44. ('12,34', None), ('1,2,3', None), ('1.234,50', None),
  45. ('123元', None), ('¥123', None), ('123RMB', None),
  46. ('NaN', None), ('Infinity', None), ('890000000-900000000', None),
  47. ])
  48. def test_numbers_and_masked_ranges(raw, expected):
  49. assert number(raw) == (Decimal(expected) if expected is not None else None)
  50. def test_chinese_headers_and_excel_numeric_codes(tmp_path, rules):
  51. from openpyxl import Workbook
  52. p = tmp_path / 'input.xlsx'
  53. wb = Workbook()
  54. ws = wb.active
  55. ws.append(['进出口标志', '商品编码', '年度', 'ISO周', '金额(美元)', '成交数量', '申报要素'])
  56. ws.append(['进口', 8532241000, 2025, 1, 100.5, 10, ROW[-1]])
  57. wb.save(p)
  58. db, stats = ingest(config(p), rules, tmp_path)
  59. try:
  60. assert stats['included'] == 1
  61. assert Store(db).total({})['amount'] == Decimal('100.5')
  62. finally:
  63. db.close()
  64. def test_all_in_scope_unknowns_are_counted_once(tmp_path, rules):
  65. p = tmp_path / 'in.csv'
  66. texts = ['品牌:ABACUS;型号:A5E30295495R2W', '品牌:无品牌;型号:无型号',
  67. '品牌:MURATA、SAMSUNG;型号:TEST-1', ROW[-1]]
  68. with p.open('w', encoding='utf-8', newline='') as f:
  69. w = csv.writer(f); w.writerow(HEAD)
  70. for text in texts: w.writerow(ROW[:4]+['100', '10', text])
  71. db, stats = ingest(config(p), rules, tmp_path)
  72. try:
  73. assert stats['included'] == 4
  74. assert Store(db).total({})['amount'] == Decimal('400')
  75. assert sum(r['n'] for r in Store(db).aggregate(('brand',))) == 4
  76. assert stats['recognition']['brand_conflict'] >= 1
  77. assert stats['recognition']['explicit_no_brand'] == 1
  78. finally:
  79. db.close()
  80. def test_multiline_physical_line_numbers(tmp_path):
  81. p = tmp_path / 'multiline.csv'
  82. with p.open('w', encoding='utf-8', newline='') as f:
  83. w = csv.writer(f); w.writerow(HEAD)
  84. w.writerow(ROW[:-1]+['品牌:MURATA\n型号:TEST-1']); w.writerow(ROW)
  85. rows = list(iter_source(p))
  86. assert [(r[0], r[1]) for r in rows] == [(0, 1), (1, 3), (2, 4)]
  87. def test_year_range_applies_to_fact(tmp_path, rules):
  88. p = tmp_path / 'old.csv'
  89. with p.open('w', newline='') as f:
  90. w=csv.writer(f); w.writerow(HEAD); w.writerow(ROW[:2]+['2024']+ROW[3:])
  91. db, stats = ingest(config(p), rules, tmp_path)
  92. try:
  93. assert stats['included'] == 0 and stats['excluded'] == 1
  94. finally:
  95. db.close()
  96. def test_duplicate_alias_and_currency_fail_clearly(tmp_path, rules):
  97. for name, headers, match in [('duplicate', HEAD+['金额'], '重复'),
  98. ('rmb', HEAD[:4]+['金额:人民币']+HEAD[5:], '人民币')]:
  99. out=tmp_path/name; out.mkdir(); p=out/'in.csv'
  100. p.write_text(','.join(headers)+'\n', encoding='utf-8')
  101. with pytest.raises(ValueError, match=match): ingest(config(p), rules, out)
  102. def test_bad_quotes_fail_without_silent_repair(tmp_path):
  103. p=tmp_path/'bad.csv'; p.write_text(','.join(HEAD)+'\nI,8532241000,2025,1,100,10,"未闭合\n')
  104. with pytest.raises(ValueError, match='CSV 第'): list(iter_source(p))
  105. def test_long_cell_and_bom_boundary(tmp_path):
  106. p=tmp_path/'long.csv'
  107. text='中文'*70000
  108. with p.open('w', encoding='utf-8', newline='') as f:
  109. w=csv.writer(f); w.writerow(HEAD); w.writerow(ROW[:-1]+[text])
  110. assert list(iter_source(p))[1][3][-1] == text
  111. @pytest.mark.parametrize('amounts,expected', [(['0.1','0.2'],'0.3'),
  112. (['9007199254740993','1'],'9007199254740994'), (['1','bad'], None)])
  113. def test_custom_aggregate_precision_and_unknowns(tmp_path, amounts, expected):
  114. p=tmp_path/'input.csv'
  115. with p.open('w', newline='') as f:
  116. w=csv.writer(f); w.writerow(HEAD)
  117. for amount in amounts: w.writerow(ROW[:4]+[amount]+ROW[5:])
  118. result=load_processor().process(CustomsProcessRequest(str(p), str(tmp_path/'out'),
  119. operation='custom_aggregate', options=CustomsOptions(group_by=['brand'])))
  120. with sqlite3.connect(result.sqlite_path) as db:
  121. actual=db.execute('SELECT "实际金额(美元)" FROM s_自定义聚合').fetchone()[0]
  122. assert actual == (expected if expected is not None else '')
  123. @pytest.mark.parametrize('fn,expected', [('sum', '9007199254740994'), ('avg', '4503599627370497'), ('min', '1'), ('max', '9007199254740993')])
  124. def test_agent_data_read_aggregates_are_exact(fn, expected):
  125. from toolsvc.readers import read_rows
  126. workspace = store.create_workspace('精度测试')
  127. obj = store.register_upload(workspace['id'], 'numbers.csv', b'amount\n9007199254740993\n1\n')
  128. result = read_rows(obj['id'], aggregate={'metrics':[{'column':'amount','fn':fn,'alias':'value'}]})
  129. assert result['rows'][0]['values']['value'] == expected
  130. def test_agent_data_read_unknown_propagates():
  131. from toolsvc.readers import read_rows
  132. workspace = store.create_workspace('缺数测试')
  133. obj = store.register_upload(workspace['id'], 'numbers.csv', b'amount\n100\nmasked-range\n')
  134. result = read_rows(obj['id'], aggregate={'metrics':[{'column':'amount','fn':'sum','alias':'value'}]})
  135. assert result['rows'][0]['values']['value'] is None
  136. def test_processing_threads_keep_decimal_precision():
  137. from concurrent.futures import ThreadPoolExecutor
  138. from decimal import getcontext
  139. with ThreadPoolExecutor(max_workers=1) as executor:
  140. assert executor.submit(lambda:getcontext().prec).result() == 50
  141. def test_gb_encoding_after_long_ascii_prefix(tmp_path):
  142. p=tmp_path/'late.csv'
  143. with p.open('w', encoding='gb18030', newline='') as f:
  144. w=csv.writer(f); w.writerow(HEAD); w.writerow(ROW[:-1]+['A'*70000+'品牌:村田'])
  145. assert list(iter_source(p))[1][3][-1].endswith('品牌:村田')
  146. def test_usd_amount_selected_with_rmb_extra_column(tmp_path, rules):
  147. p=tmp_path/'both.csv'
  148. with p.open('w', newline='') as f:
  149. w=csv.writer(f);w.writerow(HEAD+['MONORSUM_RMB']);w.writerow(ROW+['8000'])
  150. db,stats=ingest(config(p),rules,tmp_path)
  151. try:
  152. assert stats['included']==1
  153. assert Store(db).total({})['amount']==Decimal('1234.50')
  154. finally:db.close()