| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364 |
- """Dry-run by default. Raw derived indexes update in place; stat objects are copied.
- CDA_DATA_DIR=/absolute/data .venv/bin/python scripts/upgrade_query_storage.py --apply --summaries
- Preserves original stat files, row text/hashes, 23 workbook sheets and original business lineage.
- """
- import argparse
- import hashlib
- import json
- import shutil
- import sys
- from pathlib import Path
- ROOT=Path(__file__).resolve().parents[1]
- sys.path.insert(0,str(ROOT/'tools-service'))
- from toolsvc.objects import store,new_id
- from toolsvc.config import settings
- from toolsvc.query_indexes import INDEX_VERSION,create_query_indexes,ensure_raw_calendar,quote
- from toolsvc.customs.engine.storage import connect,sha256_file
- from toolsvc.customs.engine.query_summaries import build_query_summaries
- def main():
- p=argparse.ArgumentParser();p.add_argument('--apply',action='store_true');p.add_argument('--summaries',action='store_true');p.add_argument('--object-id');args=p.parse_args()
- results=[]
- for item in store.list_objects():
- if args.object_id and item['id']!=args.object_id:continue
- source=item['meta'].get('sqlite_path')
- if not source:continue
- if item['meta'].get('lineage',{}).get('query_index_version')==INDEX_VERSION:continue
- entry=dict(object_id=item['id'],apply=args.apply,strategy='derived_raw_index' if item['kind']=='raw' else 'copy_new_stat_object',indexes=[],new_summaries=[])
- if args.apply:
- original=Path(source)
- if item['kind']=='raw':target=original
- else:
- directory=settings.runs_dir/new_id('query_upgrade')
- shutil.copytree(original.parent,directory)
- target=directory/'stat.sqlite'
- if original.name!='stat.sqlite':shutil.copy2(directory/original.name,target)
- db=connect(target)
- try:
- for table in item['tables']:
- cols=[r[1] for r in db.execute(f'PRAGMA table_info({quote(table["sqlite_name"])})')]
- if item['kind']=='raw':
- ensure_raw_calendar(db,table['sqlite_name'],cols)
- cols=[r[1] for r in db.execute(f'PRAGMA table_info({quote(table["sqlite_name"])})')]
- entry['indexes']+=create_query_indexes(db,table['sqlite_name'],cols,raw=item['kind']=='raw')
- db.execute('ANALYZE');db.commit()
- exists=db.execute("SELECT 1 FROM sqlite_master WHERE name='q_class_month'").fetchone()
- finally:db.close()
- if item['kind']=='stat':
- sheets=[dict(sheet=t['name'],table=t['sqlite_name'],rows=t['rows'],columns=t['columns']) for t in item['tables']]
- if args.summaries and item['producer']=='customs_process' and {'1.0-3.0_MLCC','5.1_存储器'} <= {t['name'] for t in item['tables']} and (directory/'working.sqlite').exists() and not exists:
- working=connect(directory/'working.sqlite')
- try:extra=build_query_summaries(working,directory)
- finally:working.close()
- sheets += [s.__dict__ for s in extra];entry['new_summaries']=[s.sheet for s in extra]
- lineage={**item['meta'].get('lineage',{}),'query_parent_object':item['id'],'query_index_version':INDEX_VERSION}
- lineage['output_hash']='sha256:'+hashlib.sha256((item['sha256']+sha256_file(target)+INDEX_VERSION).encode()).hexdigest()
- result=dict(sheets=sheets,lineage=lineage,stats=item['meta'].get('stats',{}),warnings=item['meta'].get('warnings',[]),xlsx_path=str(directory/'统计结果.xlsx') if (directory/'统计结果.xlsx').exists() else None)
- created=store.register_processed(item['workspace_id'],item['parents'][0] if item['parents'] else item['id'],item['name']+'(查询优化)',directory,result,producer='query_index_upgrade')
- entry['new_object_id']=created['id']
- results.append(entry)
- print(json.dumps(results,ensure_ascii=False,indent=2))
- if __name__=='__main__':main()
|