| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485 |
- """Shared streaming input reader for uploads and processing; never guesses row repairs."""
- from __future__ import annotations
- import codecs
- import csv
- from pathlib import Path
- MAX_FIELD_SIZE = 10 * 1024 * 1024
- def detect_encoding(path: Path) -> str:
- with path.open('rb') as handle:
- sample = handle.read(65536)
- if sample.startswith((codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
- return 'utf-32'
- if sample.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
- return 'utf-16'
- # Validate the full stream, because a long ASCII prefix says nothing about
- # Chinese bytes near the end. Incremental decoding handles chunk boundaries.
- for encoding in ('utf-8-sig', 'gb18030'):
- decoder = codecs.getincrementaldecoder(encoding)()
- try:
- with path.open('rb') as handle:
- for block in iter(lambda: handle.read(1 << 20), b''):
- decoder.decode(block, final=False)
- decoder.decode(b'', final=True)
- return encoding
- except UnicodeDecodeError:
- continue
- raise ValueError('无法识别文件编码,请使用 UTF-8、GB18030 或带 BOM 的 UTF-16')
- def iter_source(path: Path, encoding: str = 'auto', delimiter: str = 'auto'):
- """Yield (logical data row, physical end line, headers, values); header row is 0."""
- csv.field_size_limit(MAX_FIELD_SIZE)
- if path.suffix.lower() in {'.xlsx', '.xlsm'}:
- from openpyxl import load_workbook
- workbook = load_workbook(path, read_only=True, data_only=True)
- try:
- headers = None
- logical = 0
- for line, row in enumerate(workbook.worksheets[0].iter_rows(values_only=True), 1):
- values = ['' if cell is None else str(cell) for cell in row]
- if headers is None:
- if not any(values):
- continue
- headers = values
- yield 0, line, headers, None
- elif any(values):
- logical += 1
- yield logical, line, headers, values
- if headers is None:
- raise ValueError('输入文件为空,没有表头')
- finally:
- workbook.close()
- return
- encoding = detect_encoding(path) if encoding == 'auto' else encoding
- with path.open(encoding=encoding, newline='') as handle:
- if delimiter == 'auto':
- position = handle.tell()
- first = next((line for line in handle if line.strip()), '')
- if not first:
- raise ValueError('输入文件为空,没有表头')
- # Header names contain no declaration values. Using the header avoids
- # mistaking G_MODEL's pipes or semicolons for file delimiters.
- delimiter = max((',', '\t', ';', '|'), key=lambda sep: len(next(csv.reader([first], delimiter=sep))))
- handle.seek(position)
- reader = csv.reader(handle, delimiter=delimiter, strict=True)
- headers = None
- logical = 0
- try:
- for values in reader:
- if not values or not any(value.strip() for value in values):
- continue
- if headers is None:
- headers = values
- yield 0, reader.line_num, headers, None
- else:
- logical += 1
- yield logical, reader.line_num, headers, values
- except csv.Error as error:
- raise ValueError(f'CSV 第 {reader.line_num} 行读取失败:{error};请检查引号或超长字段') from error
- if headers is None:
- raise ValueError('输入文件为空,没有表头')
|