| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197 |
- """Adversarial API inputs, independent Decimal oracles and pagination/cache regressions."""
- import base64
- import json
- import random
- import sqlite3
- from collections import OrderedDict
- from concurrent.futures import ThreadPoolExecutor
- from decimal import Decimal
- import pytest
- from fastapi.testclient import TestClient
- from toolsvc.api import app
- from toolsvc.config import settings
- from toolsvc.objects import store
- from toolsvc.readers import read_rows, ReadError
- @pytest.fixture(scope='module')
- def client():
- with TestClient(app,raise_server_exceptions=False) as c:yield c
- @pytest.fixture(scope='module')
- def rows_object():
- ws=store.create_workspace('扩展查询回归')
- return store.register_upload(ws['id'],'regression.csv',('YEAR,WEEK,MONORSUM,COUNT,G_MODEL\n'+''.join(f'2025,{i%52+1},{i}.01,{i+1},MODEL-{i:04}\n' for i in range(257))).encode())
- @pytest.mark.parametrize('change',[
- {'aggregate':{'group_by':[{}],'metrics':[{'fn':'count'}]}},
- {'aggregate':{'group_by':'YEAR','metrics':[{'fn':'count'}]}},
- {'aggregate':{'metrics':['sum']}},
- {'aggregate':{'metrics':{'fn':'count'}}},
- {'aggregate':{'sum':'MONORSUM'}},
- {'aggregate':{'group_by':[['YEAR']],'metrics':[{'fn':'count'}]}},
- {'order_by':[{'column':{},'dir':'asc'}]},
- {'order_by':[{'column':'YEAR','dir':'sideways'}]},
- {'filters':[{'column':'YEAR','op':'in','value':[]}]},
- {'filters':[{'column':'YEAR','op':'in','value':list(range(501))}]},
- {'filters':[{'column':'YEAR','op':'between','value':[2025,2026]}]},
- {'filters':[{'column':'YEAR','value':2025}]*33},
- {'count_mode':'sometimes'},
- {'offset':-1},
- {'offset':2**80},
- {'limit':0},
- {'limit':-3},
- ])
- def test_invalid_api_queries_return_client_error_not_500(client,rows_object,change):
- response=client.post('/tools/data_read',json={'object_id':rows_object['id'],**change})
- assert response.status_code in (400,422),response.text
- @pytest.mark.parametrize('cursor',['%%%','e30=',base64.urlsafe_b64encode(b'[]').decode(),'A'*5000])
- def test_bad_cursor_shapes(client,rows_object,cursor):
- response=client.post('/tools/data_read',json={'object_id':rows_object['id'],'cursor':cursor})
- assert response.status_code==400
- @pytest.mark.parametrize('page_size',[1,2,17,200])
- def test_all_page_boundaries_and_filtered_cursor(rows_object,page_size):
- args=dict(object_id=rows_object['id'],filters=[{'column':'WEEK','op':'>=','value':25}],limit=page_size,use_cache=False)
- seen=[]; result=read_rows(**args)
- while True:
- seen.extend(r['_trace']['row_index'] for r in result['rows'])
- if result['next_cursor'] is None:break
- result=read_rows(**args,cursor=result['next_cursor'])
- expected=[i for i in range(257) if i%52+1>=25]
- assert seen==expected and len(seen)==len(set(seen))
- assert result['exhausted'] and not result['truncated']
- def test_byte_limited_cursor_does_not_skip_consumed_lookahead(rows_object,monkeypatch):
- monkeypatch.setattr(settings,'query_max_bytes',360)
- args=dict(object_id=rows_object['id'],columns=['G_MODEL'],limit=200,use_cache=False)
- seen=[];result=read_rows(**args)
- assert result['truncation_reason']=='byte_limit'
- while True:
- seen.extend(r['_trace']['row_index'] for r in result['rows'])
- if result['next_cursor'] is None:break
- result=read_rows(**args,cursor=result['next_cursor'])
- assert seen==list(range(257))
- @pytest.mark.parametrize('mode',['auto','none','exact'])
- def test_empty_filtered_results_distinguish_unknown_count_from_zero(rows_object,mode):
- q=read_rows(rows_object['id'],filters=[{'column':'G_MODEL','value':'absent'}],count_mode=mode)
- assert q['rows']==[] and q['exhausted'] and q['next_cursor'] is None
- assert q['total']==(0 if mode=='exact' else None)
- @pytest.mark.parametrize('seed',[7,31,197])
- def test_random_decimal_aggregates_match_independent_python_oracle(seed):
- rng=random.Random(seed)
- groups={str(g):[] for g in range(7)};records=[]
- for i in range(350):
- value=Decimal(rng.randrange(-10**18,10**18))/Decimal(1000)
- group=str(i%7);groups[group].append(value);records.append(f'{group},{value}')
- ws=store.create_workspace('decimal oracle')
- obj=store.register_upload(ws['id'],'decimal.csv',('group,amount\n'+'\n'.join(records)+'\n').encode())
- result=read_rows(obj['id'],aggregate={'group_by':['group'],'metrics':[{'fn':fn,'column':'amount','alias':fn} for fn in ('sum','min','max','avg')]},limit=20)
- for row in result['rows']:
- data=row['values'];values=groups[data['group']]
- assert Decimal(data['sum'])==sum(values)
- assert Decimal(data['min'])==min(values)
- assert Decimal(data['max'])==max(values)
- assert Decimal(data['avg'])==sum(values)/len(values)
- assert not result['truncated']
- @pytest.mark.parametrize('fn',['sum','avg','min','max'])
- def test_unknown_member_propagates_for_each_exact_aggregate(fn):
- ws=store.create_workspace('unknown')
- obj=store.register_upload(ws['id'],'bad.csv',b'g,amount\na,1.5\na,masked-range\nb,0\n')
- result=read_rows(obj['id'],aggregate={'group_by':['g'],'metrics':[{'fn':fn,'column':'amount','alias':'v'}]})
- values={r['values']['g']:r['values']['v'] for r in result['rows']}
- assert values=={'a':None,'b':'0'}
- def test_parallel_cache_reads_are_isolated(rows_object):
- def query(_):return read_rows(rows_object['id'],limit=11)
- with ThreadPoolExecutor(max_workers=8) as pool: results=list(pool.map(query,range(32)))
- expected=[r['_trace']['row_hash'] for r in results[0]['rows']]
- assert all([r['_trace']['row_hash'] for r in q['rows']]==expected for q in results)
- results[0]['rows'][0]['values']['G_MODEL']='mutated'
- assert read_rows(rows_object['id'],limit=11)['rows'][0]['values']['G_MODEL']=='MODEL-0000'
- def test_cache_eviction_is_bounded(rows_object,monkeypatch):
- import toolsvc.readers as readers
- monkeypatch.setattr(readers,'_CACHE',OrderedDict());monkeypatch.setattr(readers,'_CACHE_BYTES',0)
- for offset in range(70):read_rows(rows_object['id'],limit=2,offset=offset)
- assert len(readers._CACHE)<=64 and readers._CACHE_BYTES<=16*1024*1024
- assert not read_rows(rows_object['id'],limit=2,offset=0)['cache_hit']
- def test_timeout_http_408_then_connection_recovers(client,rows_object,monkeypatch):
- monkeypatch.setattr(settings,'query_timeout_ms',0)
- q={'object_id':rows_object['id'],'count_mode':'exact','use_cache':False}
- assert client.post('/tools/data_read',json=q).status_code==408
- monkeypatch.setattr(settings,'query_timeout_ms',5000)
- good=client.post('/tools/data_read',json=q)
- assert good.status_code==200 and good.json()['total']==257
- def test_cache_hits_still_record_distinct_data_plane_calls(client,rows_object):
- q={'object_id':rows_object['id'],'limit':3,'session_id':'cache-ledger-regression'}
- first=client.post('/tools/data_read',json={**q,'call_id':'regression_read_1'}).json()
- second=client.post('/tools/data_read',json={**q,'call_id':'regression_read_2'}).json()
- assert second['cache_hit'] and first['rows']==second['rows']
- calls=client.get('/ledger/calls',params={'session_id':q['session_id']}).json()['items']
- assert {'regression_read_1','regression_read_2'} <= {c['call_id'] for c in calls}
- def test_sql_injection_is_a_literal_filter_and_cannot_change_schema(client,rows_object):
- q={'object_id':rows_object['id'],'filters':[{'column':'G_MODEL','value':"' OR 1=1 --"}],'count_mode':'exact'}
- result=client.post('/tools/data_read',json=q)
- assert result.status_code==200 and result.json()['total']==0
- q['filters']=[{'column':'G_MODEL); DROP TABLE raw;--','value':'x'}]
- assert client.post('/tools/data_read',json=q).status_code==400
- assert read_rows(rows_object['id'],limit=1)['total']==257
- @pytest.fixture(scope='module')
- def stat_object(processed,sample_csv):
- ws=store.create_workspace('nonadditive policy')
- source=store.register_upload(ws['id'],'sample.csv',sample_csv.read_bytes())
- return store.register_processed(ws['id'],source['id'],'stat',processed['output'],processed['result'].to_dict())
- @pytest.mark.parametrize('metric',['价格指数','金额指数','数量指数','金额占比','数量占比','金额环比','数量环比','平均单价(美元/成交数量单位)'])
- @pytest.mark.parametrize('fn',['sum','avg'])
- def test_all_nonadditive_business_metrics_rejected(client,stat_object,metric,fn):
- q={'object_id':stat_object['id'],'table':'1.0-3.0_MLCC','aggregate':{'metrics':[{'fn':fn,'column':metric}]}}
- response=client.post('/tools/data_read',json=q)
- assert response.status_code==400 and '不能' in response.json()['detail']
- def test_external_masked_stat_has_no_automatic_additivity(client):
- ws=store.create_workspace('external masked')
- obj=store.register_upload(ws['id'],'已遮掩.csv','实际金额(美元)\n100-200\n'.encode(),kind='stat')
- response=client.post('/tools/data_read',json={'object_id':obj['id'],'aggregate':{'metrics':[{'fn':'sum','column':'实际金额(美元)'}]}})
- assert response.status_code==400
- def test_browser_fixture_totals_are_a_fixed_oracle(tmp_path,root):
- from toolsvc.customs import CustomsOptions,CustomsProcessRequest,load_processor
- from toolsvc.customs.engine.numbers import StrictSum
- expected=json.loads((root/'samples/ui_regression.expected.json').read_text())
- result=load_processor().process(CustomsProcessRequest(str(root/'samples/ui_regression.csv'),str(tmp_path/'ui'),options=CustomsOptions(directions=['I'],cutoff_date='2026-12-31')))
- assert [result.stats[k] for k in ('total','included','excluded')]==[63,60,3]
- with sqlite3.connect(result.sqlite_path) as db:
- db.create_aggregate('dsum',1,StrictSum)
- for table in ('q_class_month','q_model_month'):
- amount,quantity,n=db.execute(f'SELECT dsum("实际金额(美元)"),dsum("成交数量"),SUM(CAST("记录数" AS INTEGER)) FROM {table}').fetchone()
- assert Decimal(amount)==Decimal(expected['amount_usd'])
- assert Decimal(quantity)==Decimal(expected['quantity'])
- assert n==expected['included']
|