"""Adversarial API inputs, independent Decimal oracles and pagination/cache regressions.""" import base64 import json import random import sqlite3 from collections import OrderedDict from concurrent.futures import ThreadPoolExecutor from decimal import Decimal import pytest from fastapi.testclient import TestClient from toolsvc.api import app from toolsvc.config import settings from toolsvc.objects import store from toolsvc.readers import read_rows, ReadError @pytest.fixture(scope='module') def client(): with TestClient(app,raise_server_exceptions=False) as c:yield c @pytest.fixture(scope='module') def rows_object(): ws=store.create_workspace('扩展查询回归') return store.register_upload(ws['id'],'regression.csv',('YEAR,WEEK,MONORSUM,COUNT,G_MODEL\n'+''.join(f'2025,{i%52+1},{i}.01,{i+1},MODEL-{i:04}\n' for i in range(257))).encode()) @pytest.mark.parametrize('change',[ {'aggregate':{'group_by':[{}],'metrics':[{'fn':'count'}]}}, {'aggregate':{'group_by':'YEAR','metrics':[{'fn':'count'}]}}, {'aggregate':{'metrics':['sum']}}, {'aggregate':{'metrics':{'fn':'count'}}}, {'aggregate':{'sum':'MONORSUM'}}, {'aggregate':{'group_by':[['YEAR']],'metrics':[{'fn':'count'}]}}, {'order_by':[{'column':{},'dir':'asc'}]}, {'order_by':[{'column':'YEAR','dir':'sideways'}]}, {'filters':[{'column':'YEAR','op':'in','value':[]}]}, {'filters':[{'column':'YEAR','op':'in','value':list(range(501))}]}, {'filters':[{'column':'YEAR','op':'between','value':[2025,2026]}]}, {'filters':[{'column':'YEAR','value':2025}]*33}, {'count_mode':'sometimes'}, {'offset':-1}, {'offset':2**80}, {'limit':0}, {'limit':-3}, ]) def test_invalid_api_queries_return_client_error_not_500(client,rows_object,change): response=client.post('/tools/data_read',json={'object_id':rows_object['id'],**change}) assert response.status_code in (400,422),response.text @pytest.mark.parametrize('cursor',['%%%','e30=',base64.urlsafe_b64encode(b'[]').decode(),'A'*5000]) def test_bad_cursor_shapes(client,rows_object,cursor): response=client.post('/tools/data_read',json={'object_id':rows_object['id'],'cursor':cursor}) assert response.status_code==400 @pytest.mark.parametrize('page_size',[1,2,17,200]) def test_all_page_boundaries_and_filtered_cursor(rows_object,page_size): args=dict(object_id=rows_object['id'],filters=[{'column':'WEEK','op':'>=','value':25}],limit=page_size,use_cache=False) seen=[]; result=read_rows(**args) while True: seen.extend(r['_trace']['row_index'] for r in result['rows']) if result['next_cursor'] is None:break result=read_rows(**args,cursor=result['next_cursor']) expected=[i for i in range(257) if i%52+1>=25] assert seen==expected and len(seen)==len(set(seen)) assert result['exhausted'] and not result['truncated'] def test_byte_limited_cursor_does_not_skip_consumed_lookahead(rows_object,monkeypatch): monkeypatch.setattr(settings,'query_max_bytes',360) args=dict(object_id=rows_object['id'],columns=['G_MODEL'],limit=200,use_cache=False) seen=[];result=read_rows(**args) assert result['truncation_reason']=='byte_limit' while True: seen.extend(r['_trace']['row_index'] for r in result['rows']) if result['next_cursor'] is None:break result=read_rows(**args,cursor=result['next_cursor']) assert seen==list(range(257)) @pytest.mark.parametrize('mode',['auto','none','exact']) def test_empty_filtered_results_distinguish_unknown_count_from_zero(rows_object,mode): q=read_rows(rows_object['id'],filters=[{'column':'G_MODEL','value':'absent'}],count_mode=mode) assert q['rows']==[] and q['exhausted'] and q['next_cursor'] is None assert q['total']==(0 if mode=='exact' else None) @pytest.mark.parametrize('seed',[7,31,197]) def test_random_decimal_aggregates_match_independent_python_oracle(seed): rng=random.Random(seed) groups={str(g):[] for g in range(7)};records=[] for i in range(350): value=Decimal(rng.randrange(-10**18,10**18))/Decimal(1000) group=str(i%7);groups[group].append(value);records.append(f'{group},{value}') ws=store.create_workspace('decimal oracle') obj=store.register_upload(ws['id'],'decimal.csv',('group,amount\n'+'\n'.join(records)+'\n').encode()) result=read_rows(obj['id'],aggregate={'group_by':['group'],'metrics':[{'fn':fn,'column':'amount','alias':fn} for fn in ('sum','min','max','avg')]},limit=20) for row in result['rows']: data=row['values'];values=groups[data['group']] assert Decimal(data['sum'])==sum(values) assert Decimal(data['min'])==min(values) assert Decimal(data['max'])==max(values) assert Decimal(data['avg'])==sum(values)/len(values) assert not result['truncated'] @pytest.mark.parametrize('fn',['sum','avg','min','max']) def test_unknown_member_propagates_for_each_exact_aggregate(fn): ws=store.create_workspace('unknown') obj=store.register_upload(ws['id'],'bad.csv',b'g,amount\na,1.5\na,masked-range\nb,0\n') result=read_rows(obj['id'],aggregate={'group_by':['g'],'metrics':[{'fn':fn,'column':'amount','alias':'v'}]}) values={r['values']['g']:r['values']['v'] for r in result['rows']} assert values=={'a':None,'b':'0'} def test_parallel_cache_reads_are_isolated(rows_object): def query(_):return read_rows(rows_object['id'],limit=11) with ThreadPoolExecutor(max_workers=8) as pool: results=list(pool.map(query,range(32))) expected=[r['_trace']['row_hash'] for r in results[0]['rows']] assert all([r['_trace']['row_hash'] for r in q['rows']]==expected for q in results) results[0]['rows'][0]['values']['G_MODEL']='mutated' assert read_rows(rows_object['id'],limit=11)['rows'][0]['values']['G_MODEL']=='MODEL-0000' def test_cache_eviction_is_bounded(rows_object,monkeypatch): import toolsvc.readers as readers monkeypatch.setattr(readers,'_CACHE',OrderedDict());monkeypatch.setattr(readers,'_CACHE_BYTES',0) for offset in range(70):read_rows(rows_object['id'],limit=2,offset=offset) assert len(readers._CACHE)<=64 and readers._CACHE_BYTES<=16*1024*1024 assert not read_rows(rows_object['id'],limit=2,offset=0)['cache_hit'] def test_timeout_http_408_then_connection_recovers(client,rows_object,monkeypatch): monkeypatch.setattr(settings,'query_timeout_ms',0) q={'object_id':rows_object['id'],'count_mode':'exact','use_cache':False} assert client.post('/tools/data_read',json=q).status_code==408 monkeypatch.setattr(settings,'query_timeout_ms',5000) good=client.post('/tools/data_read',json=q) assert good.status_code==200 and good.json()['total']==257 def test_cache_hits_still_record_distinct_data_plane_calls(client,rows_object): q={'object_id':rows_object['id'],'limit':3,'session_id':'cache-ledger-regression'} first=client.post('/tools/data_read',json={**q,'call_id':'regression_read_1'}).json() second=client.post('/tools/data_read',json={**q,'call_id':'regression_read_2'}).json() assert second['cache_hit'] and first['rows']==second['rows'] calls=client.get('/ledger/calls',params={'session_id':q['session_id']}).json()['items'] assert {'regression_read_1','regression_read_2'} <= {c['call_id'] for c in calls} def test_sql_injection_is_a_literal_filter_and_cannot_change_schema(client,rows_object): q={'object_id':rows_object['id'],'filters':[{'column':'G_MODEL','value':"' OR 1=1 --"}],'count_mode':'exact'} result=client.post('/tools/data_read',json=q) assert result.status_code==200 and result.json()['total']==0 q['filters']=[{'column':'G_MODEL); DROP TABLE raw;--','value':'x'}] assert client.post('/tools/data_read',json=q).status_code==400 assert read_rows(rows_object['id'],limit=1)['total']==257 @pytest.fixture(scope='module') def stat_object(processed,sample_csv): ws=store.create_workspace('nonadditive policy') source=store.register_upload(ws['id'],'sample.csv',sample_csv.read_bytes()) return store.register_processed(ws['id'],source['id'],'stat',processed['output'],processed['result'].to_dict()) @pytest.mark.parametrize('metric',['价格指数','金额指数','数量指数','金额占比','数量占比','金额环比','数量环比','平均单价(美元/成交数量单位)']) @pytest.mark.parametrize('fn',['sum','avg']) def test_all_nonadditive_business_metrics_rejected(client,stat_object,metric,fn): q={'object_id':stat_object['id'],'table':'1.0-3.0_MLCC','aggregate':{'metrics':[{'fn':fn,'column':metric}]}} response=client.post('/tools/data_read',json=q) assert response.status_code==400 and '不能' in response.json()['detail'] def test_external_masked_stat_has_no_automatic_additivity(client): ws=store.create_workspace('external masked') obj=store.register_upload(ws['id'],'已遮掩.csv','实际金额(美元)\n100-200\n'.encode(),kind='stat') response=client.post('/tools/data_read',json={'object_id':obj['id'],'aggregate':{'metrics':[{'fn':'sum','column':'实际金额(美元)'}]}}) assert response.status_code==400 def test_browser_fixture_totals_are_a_fixed_oracle(tmp_path,root): from toolsvc.customs import CustomsOptions,CustomsProcessRequest,load_processor from toolsvc.customs.engine.numbers import StrictSum expected=json.loads((root/'samples/ui_regression.expected.json').read_text()) result=load_processor().process(CustomsProcessRequest(str(root/'samples/ui_regression.csv'),str(tmp_path/'ui'),options=CustomsOptions(directions=['I'],cutoff_date='2026-12-31'))) assert [result.stats[k] for k in ('total','included','excluded')]==[63,60,3] with sqlite3.connect(result.sqlite_path) as db: db.create_aggregate('dsum',1,StrictSum) for table in ('q_class_month','q_model_month'): amount,quantity,n=db.execute(f'SELECT dsum("实际金额(美元)"),dsum("成交数量"),SUM(CAST("记录数" AS INTEGER)) FROM {table}').fetchone() assert Decimal(amount)==Decimal(expected['amount_usd']) assert Decimal(quantity)==Decimal(expected['quantity']) assert n==expected['included']