| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081 |
- """Reproducible comparison with the supplied legacy parser and ingestion benchmark.
- Usage: .venv/bin/python scripts/qualify_tool.py --legacy ../reference_20261117/component_trade_promax --rows 100000
- All generated data is synthetic. Timings do not estimate production throughput.
- """
- import argparse
- import csv
- import importlib.util
- import json
- import sys
- import time
- from pathlib import Path
- from decimal import Decimal
- ROOT=Path(__file__).resolve().parents[1]
- sys.path.insert(0,str(ROOT/'tools-service'))
- from toolsvc.customs.engine.rules import Rules
- from toolsvc.customs.engine.storage import ingest, Store
- def main():
- parser=argparse.ArgumentParser()
- parser.add_argument('--legacy',type=Path,required=True)
- parser.add_argument('--rows',type=int,default=100000)
- parser.add_argument('--before',type=Path)
- args=parser.parse_args()
- out=ROOT/'.run/qualification'; out.mkdir(parents=True,exist_ok=True)
- folder=ROOT/'tools-service/toolsvc/customs/rules'
- mapping=folder/'数据映射规则.xlsx'; fields=folder/'进出口数据字段需求20260819.xlsx'
- sys.path.insert(0,str(args.legacy.resolve()))
- from trade0909.rules import Rules as LegacyRules
- legacy=LegacyRules(mapping, fields); current=Rules(mapping, fields)
- # Independent baseline: agent implementation before this change.
- original=args.before
- before=None
- if original and original.exists():
- spec=importlib.util.spec_from_file_location('before_rules',original)
- mod=importlib.util.module_from_spec(spec); spec.loader.exec_module(mod)
- before=mod.Rules(mapping, fields)
- texts=[
- '4|0|多层|陶瓷|MURATA|TEST-1|||',
- '品牌:MURATA;型号:TEST-1', '品牌名称:村田;型号规格:TEST-1',
- '村田(MURATA)|TEST-1', 'MURATA/TEST-1/计算机用',
- 'MURATA_TEST-1', '0 0 多层片式 瓷介 MURATA TEST-1',
- '品牌为MURATA,型号为TEST-1', '(品牌)MURATA(型号)TEST-1',
- '品牌|MURATA|型号|TEST-1', '品牌:MURTA;型号:TEST-1',
- '品牌:MURATA,SAMSUNG;型号:TEST-1',
- '品牌:ABACUS;型号:A5E30295495R2W', '品牌:无品牌;型号:无型号',
- '品牌:MURATA;型号:1/10W', '品牌:MURATA;用途:5G基站',
- ]
- records=[]
- keys=['brand','model','brand_country','c1','c2','c3','c4','app','mapped']
- for text in texts:
- old=legacy.parse('8532241000',text); new=current.parse('8532241000',text)
- prev=before.parse('8532241000',text) if before else None
- records.append(dict(raw=text,legacy={k:old[k] for k in keys},current={k:new[k] for k in keys},
- legacy_equal=all(old[k]==new[k] for k in keys),
- before_pair=[prev['brand'],prev['model']] if prev else None))
- source=out/'synthetic.csv'
- with source.open('w',encoding='utf-8-sig',newline='') as f:
- w=csv.writer(f);w.writerow(['I_E_FLAG','HS_CODE','YEAR','WEEK','MONORSUM','COUNT','G_MODEL'])
- for i in range(args.rows):
- w.writerow(['I','8532241000','2025',str(i%52+1),'0.10','1',texts[i%len(texts)]])
- bench=out/f'bench-{time.time_ns()}';bench.mkdir()
- start=time.perf_counter()
- db,stats=ingest(dict(input_csv=str(source),directions=['I'],start_year=2025,end_year=2026,cutoff_date='2026-12-31'),current,bench)
- elapsed=time.perf_counter()-start
- totals=Store(db).total({});db.close()
- assert stats['included']==args.rows
- assert totals['amount']==Decimal('0.1')*args.rows
- report=dict(comparison=records, equal_to_legacy=sum(r['legacy_equal'] for r in records),
- cases=len(records),before_recognized_pairs=sum(bool(r['before_pair'] and all(r['before_pair'])) for r in records),
- current_recognized_pairs=sum(bool(r['current']['brand'] and r['current']['model']) for r in records),
- synthetic_benchmark=dict(rows=args.rows,distinct_declarations=len(texts),seconds=round(elapsed,3),
- rows_per_second=round(args.rows/elapsed),amount=str(totals['amount']),included=stats['included'],
- scope='stream reading, parsing, audited SQLite ingestion, monthly aggregation and indexes; excludes full reports and uploads'))
- (out/'report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2))
- print(json.dumps({k:v for k,v in report.items() if k!='comparison'},ensure_ascii=False,indent=2))
- assert report['equal_to_legacy']==len(records)
- if __name__=='__main__':main()
|