verify_business_parity.py 3.7 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556
  1. """Run ZIP reports on the same ingested sample; compare original business CSVs exactly."""
  2. import argparse
  3. import csv
  4. from collections import Counter
  5. import json
  6. import sys
  7. import time
  8. from pathlib import Path
  9. ROOT=Path(__file__).resolve().parents[1]
  10. sys.path.insert(0,str(ROOT/'tools-service'))
  11. def main():
  12. p=argparse.ArgumentParser();p.add_argument('--legacy',type=Path,required=True);p.add_argument('--out',type=Path,default=ROOT/'.run'/('parity-'+str(time.time_ns())));args=p.parse_args()
  13. sys.path.insert(0,str(args.legacy.resolve()))
  14. from toolsvc.customs import load_processor,CustomsProcessRequest,CustomsOptions
  15. from toolsvc.customs.engine.storage import connect
  16. from toolsvc.customs.engine.reports import Reports
  17. from toolsvc.customs.engine.rules import Rules
  18. from trade0909.reports import Reports as ZIPReports
  19. out=args.out;out.mkdir(parents=True)
  20. current=load_processor().process(CustomsProcessRequest(str(ROOT/'samples/small.csv'),str(out/'current'),options=CustomsOptions(cutoff_date='2026-12-31')))
  21. config=json.loads((out/'current/run_config.json').read_text());stats=json.loads((out/'current/run_summary.json').read_text())
  22. config['encoding']='utf-8-sig';config['delimiter']=','
  23. rules=Rules(config['mapping_file'],config['field_file']);db=connect(out/'current/working.sqlite')
  24. db.execute('CREATE TABLE IF NOT EXISTS fail_shapes(reason TEXT,field TEXT,shape TEXT,n INTEGER)')
  25. legacy=out/'zip';legacy.mkdir()
  26. ZIPReports(db,config,stats,rules,legacy).run();db.close()
  27. checked=[]; differences=[]; metric_differences=[]; order_only=[]
  28. for path in sorted((out/'current/精确结果CSV').glob('*.csv')):
  29. # Audit wording and new engine/version metadata differ by design; compare business outputs.
  30. if path.stem in ('数据质量摘要','映射冲突','周四归月映射'):continue
  31. old=legacy/'精确结果CSV'/path.name
  32. if not old.exists():differences.append(dict(sheet=path.stem,error='missing ZIP sheet'));continue
  33. with path.open(encoding='utf-8-sig',newline='') as a,old.open(encoding='utf-8-sig',newline='') as b:
  34. ar=list(csv.reader(a));br=list(csv.reader(b))
  35. allowed_labels={'父级路径','占比分母范围'}
  36. def canonical(rows):
  37. return Counter(tuple('[]' if ar[0][i] in allowed_labels and value in ('MLCC','DDR') else value
  38. for i,value in enumerate(row)) for row in rows)
  39. equal_business = ar[0]==br[0] and canonical(ar[1:])==canonical(br[1:])
  40. if not equal_business:
  41. metric_differences.append(path.stem)
  42. if ar!=br:
  43. if Counter(map(tuple,ar[1:]))==Counter(map(tuple,br[1:])):
  44. order_only.append(path.stem)
  45. differences.append(dict(sheet=path.stem,current_rows=len(ar)-1,zip_rows=len(br)-1,
  46. first_differences=[dict(row=i,current=x,zip=y) for i,(x,y) in enumerate(zip(ar,br)) if x!=y][:3]))
  47. checked.append(dict(sheet=path.stem,rows=len(ar)-1,equal=ar==br))
  48. report=dict(scope='same ingested rows + same ZIP workbooks and parser; formula/output parity, not ingest-normalization parity',checked=checked,differences=differences,metric_or_scope_value_differences=metric_differences,row_order_only_tables=order_only,comparison="row multiset including duplicate multiplicities; only MLCC/DDR root label aliases normalized in two documented text fields",allowed_label_only_differences=['父级路径','占比分母范围'])
  49. (out/'report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2))
  50. print(json.dumps(dict(tables=len(checked),differences=len(differences),report=str(out/'report.json')),ensure_ascii=False))
  51. if metric_differences:raise SystemExit(1)
  52. if __name__=='__main__':main()