"""Dry-run by default. Raw derived indexes update in place; stat objects are copied. CDA_DATA_DIR=/absolute/data .venv/bin/python scripts/upgrade_query_storage.py --apply --summaries Preserves original stat files, row text/hashes, 23 workbook sheets and original business lineage. """ import argparse import hashlib import json import shutil import sys from pathlib import Path ROOT=Path(__file__).resolve().parents[1] sys.path.insert(0,str(ROOT/'tools-service')) from toolsvc.objects import store,new_id from toolsvc.config import settings from toolsvc.query_indexes import INDEX_VERSION,create_query_indexes,ensure_raw_calendar,quote from toolsvc.customs.engine.storage import connect,sha256_file from toolsvc.customs.engine.query_summaries import build_query_summaries def main(): p=argparse.ArgumentParser();p.add_argument('--apply',action='store_true');p.add_argument('--summaries',action='store_true');p.add_argument('--object-id');args=p.parse_args() results=[] for item in store.list_objects(): if args.object_id and item['id']!=args.object_id:continue source=item['meta'].get('sqlite_path') if not source:continue if item['meta'].get('lineage',{}).get('query_index_version')==INDEX_VERSION:continue entry=dict(object_id=item['id'],apply=args.apply,strategy='derived_raw_index' if item['kind']=='raw' else 'copy_new_stat_object',indexes=[],new_summaries=[]) if args.apply: original=Path(source) if item['kind']=='raw':target=original else: directory=settings.runs_dir/new_id('query_upgrade') shutil.copytree(original.parent,directory) target=directory/'stat.sqlite' if original.name!='stat.sqlite':shutil.copy2(directory/original.name,target) db=connect(target) try: for table in item['tables']: cols=[r[1] for r in db.execute(f'PRAGMA table_info({quote(table["sqlite_name"])})')] if item['kind']=='raw': ensure_raw_calendar(db,table['sqlite_name'],cols) cols=[r[1] for r in db.execute(f'PRAGMA table_info({quote(table["sqlite_name"])})')] entry['indexes']+=create_query_indexes(db,table['sqlite_name'],cols,raw=item['kind']=='raw') db.execute('ANALYZE');db.commit() exists=db.execute("SELECT 1 FROM sqlite_master WHERE name='q_class_month'").fetchone() finally:db.close() if item['kind']=='stat': sheets=[dict(sheet=t['name'],table=t['sqlite_name'],rows=t['rows'],columns=t['columns']) for t in item['tables']] if args.summaries and item['producer']=='customs_process' and {'1.0-3.0_MLCC','5.1_存储器'} <= {t['name'] for t in item['tables']} and (directory/'working.sqlite').exists() and not exists: working=connect(directory/'working.sqlite') try:extra=build_query_summaries(working,directory) finally:working.close() sheets += [s.__dict__ for s in extra];entry['new_summaries']=[s.sheet for s in extra] lineage={**item['meta'].get('lineage',{}),'query_parent_object':item['id'],'query_index_version':INDEX_VERSION} lineage['output_hash']='sha256:'+hashlib.sha256((item['sha256']+sha256_file(target)+INDEX_VERSION).encode()).hexdigest() result=dict(sheets=sheets,lineage=lineage,stats=item['meta'].get('stats',{}),warnings=item['meta'].get('warnings',[]),xlsx_path=str(directory/'统计结果.xlsx') if (directory/'统计结果.xlsx').exists() else None) created=store.register_processed(item['workspace_id'],item['parents'][0] if item['parents'] else item['id'],item['name']+'(查询优化)',directory,result,producer='query_index_upgrade') entry['new_object_id']=created['id'] results.append(entry) print(json.dumps(results,ensure_ascii=False,indent=2)) if __name__=='__main__':main()