"""Generated format matrix, accounting invariants and precision regressions.""" import csv import sqlite3 from decimal import Decimal from pathlib import Path import pytest from toolsvc.customs import CustomsOptions, CustomsProcessRequest, load_processor from toolsvc.customs.engine.numbers import number from toolsvc.customs.engine.rules import Rules from toolsvc.customs.engine.storage import Store, ingest from toolsvc.objects import _iter_rows, store from toolsvc.source import iter_source HEAD = ['I_E_FLAG', 'HS_CODE', 'YEAR', 'WEEK', 'MONORSUM_USD', 'COUNT', 'G_MODEL'] ROW = ['I', '8532241000', '2025', '1', '1234.50', '10', '品牌:MURATA;型号:TEST-1'] RULE_DIR = Path(__file__).resolve().parents[1] / 'toolsvc/customs/rules' @pytest.fixture(scope='module') def rules(): return Rules(RULE_DIR / '数据映射规则.xlsx', RULE_DIR / '进出口数据字段需求20260819.xlsx') def config(path, **extra): return dict(input_csv=str(path), start_year=2025, end_year=2026, directions=['I'], cutoff_date='2026-12-31', **extra) @pytest.mark.parametrize('encoding', ['utf-8', 'utf-8-sig', 'gb18030', 'utf-16']) @pytest.mark.parametrize('delimiter', [',', '\t', ';', '|']) def test_upload_and_ingest_same_formats(tmp_path, rules, encoding, delimiter): path = tmp_path / 'input.csv' with path.open('w', encoding=encoding, newline='') as f: w = csv.writer(f, delimiter=delimiter) w.writerow(HEAD) w.writerow(ROW) headers, iterator = _iter_rows(path) assert headers == HEAD assert list(iterator) == [ROW] db, stats = ingest(config(path), rules, tmp_path) try: assert stats['included'] == 1 assert Store(db).total({})['amount'] == Decimal('1234.50') finally: db.close() @pytest.mark.parametrize('raw,expected', [ ('1,234.50', '1234.50'), ('1,234.50', '1234.50'), ('1 234.50', '1234.50'), ('$1,234.50', '1234.50'), ('1234.50美元', '1234.50'), ('(1,234.50)', '-1234.50'), ('−100', '-100'), ('1.2345e3', '1234.5'), ('12,34', None), ('1,2,3', None), ('1.234,50', None), ('123元', None), ('¥123', None), ('123RMB', None), ('NaN', None), ('Infinity', None), ('890000000-900000000', None), ]) def test_numbers_and_masked_ranges(raw, expected): assert number(raw) == (Decimal(expected) if expected is not None else None) def test_chinese_headers_and_excel_numeric_codes(tmp_path, rules): from openpyxl import Workbook p = tmp_path / 'input.xlsx' wb = Workbook() ws = wb.active ws.append(['进出口标志', '商品编码', '年度', 'ISO周', '金额(美元)', '成交数量', '申报要素']) ws.append(['进口', 8532241000, 2025, 1, 100.5, 10, ROW[-1]]) wb.save(p) db, stats = ingest(config(p), rules, tmp_path) try: assert stats['included'] == 1 assert Store(db).total({})['amount'] == Decimal('100.5') finally: db.close() def test_all_in_scope_unknowns_are_counted_once(tmp_path, rules): p = tmp_path / 'in.csv' texts = ['品牌:ABACUS;型号:A5E30295495R2W', '品牌:无品牌;型号:无型号', '品牌:MURATA、SAMSUNG;型号:TEST-1', ROW[-1]] with p.open('w', encoding='utf-8', newline='') as f: w = csv.writer(f); w.writerow(HEAD) for text in texts: w.writerow(ROW[:4]+['100', '10', text]) db, stats = ingest(config(p), rules, tmp_path) try: assert stats['included'] == 4 assert Store(db).total({})['amount'] == Decimal('400') assert sum(r['n'] for r in Store(db).aggregate(('brand',))) == 4 assert stats['recognition']['brand_conflict'] >= 1 assert stats['recognition']['explicit_no_brand'] == 1 finally: db.close() def test_multiline_physical_line_numbers(tmp_path): p = tmp_path / 'multiline.csv' with p.open('w', encoding='utf-8', newline='') as f: w = csv.writer(f); w.writerow(HEAD) w.writerow(ROW[:-1]+['品牌:MURATA\n型号:TEST-1']); w.writerow(ROW) rows = list(iter_source(p)) assert [(r[0], r[1]) for r in rows] == [(0, 1), (1, 3), (2, 4)] def test_year_range_applies_to_fact(tmp_path, rules): p = tmp_path / 'old.csv' with p.open('w', newline='') as f: w=csv.writer(f); w.writerow(HEAD); w.writerow(ROW[:2]+['2024']+ROW[3:]) db, stats = ingest(config(p), rules, tmp_path) try: assert stats['included'] == 0 and stats['excluded'] == 1 finally: db.close() def test_duplicate_alias_and_currency_fail_clearly(tmp_path, rules): for name, headers, match in [('duplicate', HEAD+['金额'], '重复'), ('rmb', HEAD[:4]+['金额:人民币']+HEAD[5:], '人民币')]: out=tmp_path/name; out.mkdir(); p=out/'in.csv' p.write_text(','.join(headers)+'\n', encoding='utf-8') with pytest.raises(ValueError, match=match): ingest(config(p), rules, out) def test_bad_quotes_fail_without_silent_repair(tmp_path): p=tmp_path/'bad.csv'; p.write_text(','.join(HEAD)+'\nI,8532241000,2025,1,100,10,"未闭合\n') with pytest.raises(ValueError, match='CSV 第'): list(iter_source(p)) def test_long_cell_and_bom_boundary(tmp_path): p=tmp_path/'long.csv' text='中文'*70000 with p.open('w', encoding='utf-8', newline='') as f: w=csv.writer(f); w.writerow(HEAD); w.writerow(ROW[:-1]+[text]) assert list(iter_source(p))[1][3][-1] == text @pytest.mark.parametrize('amounts,expected', [(['0.1','0.2'],'0.3'), (['9007199254740993','1'],'9007199254740994'), (['1','bad'], None)]) def test_custom_aggregate_precision_and_unknowns(tmp_path, amounts, expected): p=tmp_path/'input.csv' with p.open('w', newline='') as f: w=csv.writer(f); w.writerow(HEAD) for amount in amounts: w.writerow(ROW[:4]+[amount]+ROW[5:]) result=load_processor().process(CustomsProcessRequest(str(p), str(tmp_path/'out'), operation='custom_aggregate', options=CustomsOptions(group_by=['brand']))) with sqlite3.connect(result.sqlite_path) as db: actual=db.execute('SELECT "实际金额(美元)" FROM s_自定义聚合').fetchone()[0] assert actual == (expected if expected is not None else '') @pytest.mark.parametrize('fn,expected', [('sum', '9007199254740994'), ('avg', '4503599627370497'), ('min', '1'), ('max', '9007199254740993')]) def test_agent_data_read_aggregates_are_exact(fn, expected): from toolsvc.readers import read_rows workspace = store.create_workspace('精度测试') obj = store.register_upload(workspace['id'], 'numbers.csv', b'amount\n9007199254740993\n1\n') result = read_rows(obj['id'], aggregate={'metrics':[{'column':'amount','fn':fn,'alias':'value'}]}) assert result['rows'][0]['values']['value'] == expected def test_agent_data_read_unknown_propagates(): from toolsvc.readers import read_rows workspace = store.create_workspace('缺数测试') obj = store.register_upload(workspace['id'], 'numbers.csv', b'amount\n100\nmasked-range\n') result = read_rows(obj['id'], aggregate={'metrics':[{'column':'amount','fn':'sum','alias':'value'}]}) assert result['rows'][0]['values']['value'] is None def test_processing_threads_keep_decimal_precision(): from concurrent.futures import ThreadPoolExecutor from decimal import getcontext with ThreadPoolExecutor(max_workers=1) as executor: assert executor.submit(lambda:getcontext().prec).result() == 50 def test_gb_encoding_after_long_ascii_prefix(tmp_path): p=tmp_path/'late.csv' with p.open('w', encoding='gb18030', newline='') as f: w=csv.writer(f); w.writerow(HEAD); w.writerow(ROW[:-1]+['A'*70000+'品牌:村田']) assert list(iter_source(p))[1][3][-1].endswith('品牌:村田') def test_usd_amount_selected_with_rmb_extra_column(tmp_path, rules): p=tmp_path/'both.csv' with p.open('w', newline='') as f: w=csv.writer(f);w.writerow(HEAD+['MONORSUM_RMB']);w.writerow(ROW+['8000']) db,stats=ingest(config(p),rules,tmp_path) try: assert stats['included']==1 assert Store(db).total({})['amount']==Decimal('1234.50') finally:db.close()