| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191 |
- """Generated format matrix, accounting invariants and precision regressions."""
- import csv
- import sqlite3
- from decimal import Decimal
- from pathlib import Path
- import pytest
- from toolsvc.customs import CustomsOptions, CustomsProcessRequest, load_processor
- from toolsvc.customs.engine.numbers import number
- from toolsvc.customs.engine.rules import Rules
- from toolsvc.customs.engine.storage import Store, ingest
- from toolsvc.objects import _iter_rows, store
- from toolsvc.source import iter_source
- HEAD = ['I_E_FLAG', 'HS_CODE', 'YEAR', 'WEEK', 'MONORSUM_USD', 'COUNT', 'G_MODEL']
- ROW = ['I', '8532241000', '2025', '1', '1234.50', '10', '品牌:MURATA;型号:TEST-1']
- RULE_DIR = Path(__file__).resolve().parents[1] / 'toolsvc/customs/rules'
- @pytest.fixture(scope='module')
- def rules():
- return Rules(RULE_DIR / '数据映射规则.xlsx', RULE_DIR / '进出口数据字段需求20260819.xlsx')
- def config(path, **extra):
- return dict(input_csv=str(path), start_year=2025, end_year=2026,
- directions=['I'], cutoff_date='2026-12-31', **extra)
- @pytest.mark.parametrize('encoding', ['utf-8', 'utf-8-sig', 'gb18030', 'utf-16'])
- @pytest.mark.parametrize('delimiter', [',', '\t', ';', '|'])
- def test_upload_and_ingest_same_formats(tmp_path, rules, encoding, delimiter):
- path = tmp_path / 'input.csv'
- with path.open('w', encoding=encoding, newline='') as f:
- w = csv.writer(f, delimiter=delimiter)
- w.writerow(HEAD)
- w.writerow(ROW)
- headers, iterator = _iter_rows(path)
- assert headers == HEAD
- assert list(iterator) == [ROW]
- db, stats = ingest(config(path), rules, tmp_path)
- try:
- assert stats['included'] == 1
- assert Store(db).total({})['amount'] == Decimal('1234.50')
- finally:
- db.close()
- @pytest.mark.parametrize('raw,expected', [
- ('1,234.50', '1234.50'), ('1,234.50', '1234.50'),
- ('1 234.50', '1234.50'), ('$1,234.50', '1234.50'),
- ('1234.50美元', '1234.50'), ('(1,234.50)', '-1234.50'),
- ('−100', '-100'), ('1.2345e3', '1234.5'),
- ('12,34', None), ('1,2,3', None), ('1.234,50', None),
- ('123元', None), ('¥123', None), ('123RMB', None),
- ('NaN', None), ('Infinity', None), ('890000000-900000000', None),
- ])
- def test_numbers_and_masked_ranges(raw, expected):
- assert number(raw) == (Decimal(expected) if expected is not None else None)
- def test_chinese_headers_and_excel_numeric_codes(tmp_path, rules):
- from openpyxl import Workbook
- p = tmp_path / 'input.xlsx'
- wb = Workbook()
- ws = wb.active
- ws.append(['进出口标志', '商品编码', '年度', 'ISO周', '金额(美元)', '成交数量', '申报要素'])
- ws.append(['进口', 8532241000, 2025, 1, 100.5, 10, ROW[-1]])
- wb.save(p)
- db, stats = ingest(config(p), rules, tmp_path)
- try:
- assert stats['included'] == 1
- assert Store(db).total({})['amount'] == Decimal('100.5')
- finally:
- db.close()
- def test_all_in_scope_unknowns_are_counted_once(tmp_path, rules):
- p = tmp_path / 'in.csv'
- texts = ['品牌:ABACUS;型号:A5E30295495R2W', '品牌:无品牌;型号:无型号',
- '品牌:MURATA、SAMSUNG;型号:TEST-1', ROW[-1]]
- with p.open('w', encoding='utf-8', newline='') as f:
- w = csv.writer(f); w.writerow(HEAD)
- for text in texts: w.writerow(ROW[:4]+['100', '10', text])
- db, stats = ingest(config(p), rules, tmp_path)
- try:
- assert stats['included'] == 4
- assert Store(db).total({})['amount'] == Decimal('400')
- assert sum(r['n'] for r in Store(db).aggregate(('brand',))) == 4
- assert stats['recognition']['brand_conflict'] >= 1
- assert stats['recognition']['explicit_no_brand'] == 1
- finally:
- db.close()
- def test_multiline_physical_line_numbers(tmp_path):
- p = tmp_path / 'multiline.csv'
- with p.open('w', encoding='utf-8', newline='') as f:
- w = csv.writer(f); w.writerow(HEAD)
- w.writerow(ROW[:-1]+['品牌:MURATA\n型号:TEST-1']); w.writerow(ROW)
- rows = list(iter_source(p))
- assert [(r[0], r[1]) for r in rows] == [(0, 1), (1, 3), (2, 4)]
- def test_year_range_applies_to_fact(tmp_path, rules):
- p = tmp_path / 'old.csv'
- with p.open('w', newline='') as f:
- w=csv.writer(f); w.writerow(HEAD); w.writerow(ROW[:2]+['2024']+ROW[3:])
- db, stats = ingest(config(p), rules, tmp_path)
- try:
- assert stats['included'] == 0 and stats['excluded'] == 1
- finally:
- db.close()
- def test_duplicate_alias_and_currency_fail_clearly(tmp_path, rules):
- for name, headers, match in [('duplicate', HEAD+['金额'], '重复'),
- ('rmb', HEAD[:4]+['金额:人民币']+HEAD[5:], '人民币')]:
- out=tmp_path/name; out.mkdir(); p=out/'in.csv'
- p.write_text(','.join(headers)+'\n', encoding='utf-8')
- with pytest.raises(ValueError, match=match): ingest(config(p), rules, out)
- def test_bad_quotes_fail_without_silent_repair(tmp_path):
- p=tmp_path/'bad.csv'; p.write_text(','.join(HEAD)+'\nI,8532241000,2025,1,100,10,"未闭合\n')
- with pytest.raises(ValueError, match='CSV 第'): list(iter_source(p))
- def test_long_cell_and_bom_boundary(tmp_path):
- p=tmp_path/'long.csv'
- text='中文'*70000
- with p.open('w', encoding='utf-8', newline='') as f:
- w=csv.writer(f); w.writerow(HEAD); w.writerow(ROW[:-1]+[text])
- assert list(iter_source(p))[1][3][-1] == text
- @pytest.mark.parametrize('amounts,expected', [(['0.1','0.2'],'0.3'),
- (['9007199254740993','1'],'9007199254740994'), (['1','bad'], None)])
- def test_custom_aggregate_precision_and_unknowns(tmp_path, amounts, expected):
- p=tmp_path/'input.csv'
- with p.open('w', newline='') as f:
- w=csv.writer(f); w.writerow(HEAD)
- for amount in amounts: w.writerow(ROW[:4]+[amount]+ROW[5:])
- result=load_processor().process(CustomsProcessRequest(str(p), str(tmp_path/'out'),
- operation='custom_aggregate', options=CustomsOptions(group_by=['brand'])))
- with sqlite3.connect(result.sqlite_path) as db:
- actual=db.execute('SELECT "实际金额(美元)" FROM s_自定义聚合').fetchone()[0]
- assert actual == (expected if expected is not None else '')
- @pytest.mark.parametrize('fn,expected', [('sum', '9007199254740994'), ('avg', '4503599627370497'), ('min', '1'), ('max', '9007199254740993')])
- def test_agent_data_read_aggregates_are_exact(fn, expected):
- from toolsvc.readers import read_rows
- workspace = store.create_workspace('精度测试')
- obj = store.register_upload(workspace['id'], 'numbers.csv', b'amount\n9007199254740993\n1\n')
- result = read_rows(obj['id'], aggregate={'metrics':[{'column':'amount','fn':fn,'alias':'value'}]})
- assert result['rows'][0]['values']['value'] == expected
- def test_agent_data_read_unknown_propagates():
- from toolsvc.readers import read_rows
- workspace = store.create_workspace('缺数测试')
- obj = store.register_upload(workspace['id'], 'numbers.csv', b'amount\n100\nmasked-range\n')
- result = read_rows(obj['id'], aggregate={'metrics':[{'column':'amount','fn':'sum','alias':'value'}]})
- assert result['rows'][0]['values']['value'] is None
- def test_processing_threads_keep_decimal_precision():
- from concurrent.futures import ThreadPoolExecutor
- from decimal import getcontext
- with ThreadPoolExecutor(max_workers=1) as executor:
- assert executor.submit(lambda:getcontext().prec).result() == 50
- def test_gb_encoding_after_long_ascii_prefix(tmp_path):
- p=tmp_path/'late.csv'
- with p.open('w', encoding='gb18030', newline='') as f:
- w=csv.writer(f); w.writerow(HEAD); w.writerow(ROW[:-1]+['A'*70000+'品牌:村田'])
- assert list(iter_source(p))[1][3][-1].endswith('品牌:村田')
- def test_usd_amount_selected_with_rmb_extra_column(tmp_path, rules):
- p=tmp_path/'both.csv'
- with p.open('w', newline='') as f:
- w=csv.writer(f);w.writerow(HEAD+['MONORSUM_RMB']);w.writerow(ROW+['8000'])
- db,stats=ingest(config(p),rules,tmp_path)
- try:
- assert stats['included']==1
- assert Store(db).total({})['amount']==Decimal('1234.50')
- finally:db.close()
|