"""Shared streaming input reader for uploads and processing; never guesses row repairs.""" from __future__ import annotations import codecs import csv from pathlib import Path MAX_FIELD_SIZE = 10 * 1024 * 1024 def detect_encoding(path: Path) -> str: with path.open('rb') as handle: sample = handle.read(65536) if sample.startswith((codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)): return 'utf-32' if sample.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)): return 'utf-16' # Validate the full stream, because a long ASCII prefix says nothing about # Chinese bytes near the end. Incremental decoding handles chunk boundaries. for encoding in ('utf-8-sig', 'gb18030'): decoder = codecs.getincrementaldecoder(encoding)() try: with path.open('rb') as handle: for block in iter(lambda: handle.read(1 << 20), b''): decoder.decode(block, final=False) decoder.decode(b'', final=True) return encoding except UnicodeDecodeError: continue raise ValueError('无法识别文件编码,请使用 UTF-8、GB18030 或带 BOM 的 UTF-16') def iter_source(path: Path, encoding: str = 'auto', delimiter: str = 'auto'): """Yield (logical data row, physical end line, headers, values); header row is 0.""" csv.field_size_limit(MAX_FIELD_SIZE) if path.suffix.lower() in {'.xlsx', '.xlsm'}: from openpyxl import load_workbook workbook = load_workbook(path, read_only=True, data_only=True) try: headers = None logical = 0 for line, row in enumerate(workbook.worksheets[0].iter_rows(values_only=True), 1): values = ['' if cell is None else str(cell) for cell in row] if headers is None: if not any(values): continue headers = values yield 0, line, headers, None elif any(values): logical += 1 yield logical, line, headers, values if headers is None: raise ValueError('输入文件为空,没有表头') finally: workbook.close() return encoding = detect_encoding(path) if encoding == 'auto' else encoding with path.open(encoding=encoding, newline='') as handle: if delimiter == 'auto': position = handle.tell() first = next((line for line in handle if line.strip()), '') if not first: raise ValueError('输入文件为空,没有表头') # Header names contain no declaration values. Using the header avoids # mistaking G_MODEL's pipes or semicolons for file delimiters. delimiter = max((',', '\t', ';', '|'), key=lambda sep: len(next(csv.reader([first], delimiter=sep)))) handle.seek(position) reader = csv.reader(handle, delimiter=delimiter, strict=True) headers = None logical = 0 try: for values in reader: if not values or not any(value.strip() for value in values): continue if headers is None: headers = values yield 0, reader.line_num, headers, None else: logical += 1 yield logical, reader.line_num, headers, values except csv.Error as error: raise ValueError(f'CSV 第 {reader.line_num} 行读取失败:{error};请检查引号或超长字段') from error if headers is None: raise ValueError('输入文件为空,没有表头')