| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556 |
- """Run ZIP reports on the same ingested sample; compare original business CSVs exactly."""
- import argparse
- import csv
- from collections import Counter
- import json
- import sys
- import time
- from pathlib import Path
- ROOT=Path(__file__).resolve().parents[1]
- sys.path.insert(0,str(ROOT/'tools-service'))
- def main():
- p=argparse.ArgumentParser();p.add_argument('--legacy',type=Path,required=True);p.add_argument('--out',type=Path,default=ROOT/'.run'/('parity-'+str(time.time_ns())));args=p.parse_args()
- sys.path.insert(0,str(args.legacy.resolve()))
- from toolsvc.customs import load_processor,CustomsProcessRequest,CustomsOptions
- from toolsvc.customs.engine.storage import connect
- from toolsvc.customs.engine.reports import Reports
- from toolsvc.customs.engine.rules import Rules
- from trade0909.reports import Reports as ZIPReports
- out=args.out;out.mkdir(parents=True)
- current=load_processor().process(CustomsProcessRequest(str(ROOT/'samples/small.csv'),str(out/'current'),options=CustomsOptions(cutoff_date='2026-12-31')))
- config=json.loads((out/'current/run_config.json').read_text());stats=json.loads((out/'current/run_summary.json').read_text())
- config['encoding']='utf-8-sig';config['delimiter']=','
- rules=Rules(config['mapping_file'],config['field_file']);db=connect(out/'current/working.sqlite')
- db.execute('CREATE TABLE IF NOT EXISTS fail_shapes(reason TEXT,field TEXT,shape TEXT,n INTEGER)')
- legacy=out/'zip';legacy.mkdir()
- ZIPReports(db,config,stats,rules,legacy).run();db.close()
- checked=[]; differences=[]; metric_differences=[]; order_only=[]
- for path in sorted((out/'current/精确结果CSV').glob('*.csv')):
- # Audit wording and new engine/version metadata differ by design; compare business outputs.
- if path.stem in ('数据质量摘要','映射冲突','周四归月映射'):continue
- old=legacy/'精确结果CSV'/path.name
- if not old.exists():differences.append(dict(sheet=path.stem,error='missing ZIP sheet'));continue
- with path.open(encoding='utf-8-sig',newline='') as a,old.open(encoding='utf-8-sig',newline='') as b:
- ar=list(csv.reader(a));br=list(csv.reader(b))
- allowed_labels={'父级路径','占比分母范围'}
- def canonical(rows):
- return Counter(tuple('[]' if ar[0][i] in allowed_labels and value in ('MLCC','DDR') else value
- for i,value in enumerate(row)) for row in rows)
- equal_business = ar[0]==br[0] and canonical(ar[1:])==canonical(br[1:])
- if not equal_business:
- metric_differences.append(path.stem)
- if ar!=br:
- if Counter(map(tuple,ar[1:]))==Counter(map(tuple,br[1:])):
- order_only.append(path.stem)
- differences.append(dict(sheet=path.stem,current_rows=len(ar)-1,zip_rows=len(br)-1,
- first_differences=[dict(row=i,current=x,zip=y) for i,(x,y) in enumerate(zip(ar,br)) if x!=y][:3]))
- checked.append(dict(sheet=path.stem,rows=len(ar)-1,equal=ar==br))
- report=dict(scope='same ingested rows + same ZIP workbooks and parser; formula/output parity, not ingest-normalization parity',checked=checked,differences=differences,metric_or_scope_value_differences=metric_differences,row_order_only_tables=order_only,comparison="row multiset including duplicate multiplicities; only MLCC/DDR root label aliases normalized in two documented text fields",allowed_label_only_differences=['父级路径','占比分母范围'])
- (out/'report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2))
- print(json.dumps(dict(tables=len(checked),differences=len(differences),report=str(out/'report.json')),ensure_ascii=False))
- if metric_differences:raise SystemExit(1)
- if __name__=='__main__':main()
|