source.py 3.7 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485
  1. """Shared streaming input reader for uploads and processing; never guesses row repairs."""
  2. from __future__ import annotations
  3. import codecs
  4. import csv
  5. from pathlib import Path
  6. MAX_FIELD_SIZE = 10 * 1024 * 1024
  7. def detect_encoding(path: Path) -> str:
  8. with path.open('rb') as handle:
  9. sample = handle.read(65536)
  10. if sample.startswith((codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
  11. return 'utf-32'
  12. if sample.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
  13. return 'utf-16'
  14. # Validate the full stream, because a long ASCII prefix says nothing about
  15. # Chinese bytes near the end. Incremental decoding handles chunk boundaries.
  16. for encoding in ('utf-8-sig', 'gb18030'):
  17. decoder = codecs.getincrementaldecoder(encoding)()
  18. try:
  19. with path.open('rb') as handle:
  20. for block in iter(lambda: handle.read(1 << 20), b''):
  21. decoder.decode(block, final=False)
  22. decoder.decode(b'', final=True)
  23. return encoding
  24. except UnicodeDecodeError:
  25. continue
  26. raise ValueError('无法识别文件编码,请使用 UTF-8、GB18030 或带 BOM 的 UTF-16')
  27. def iter_source(path: Path, encoding: str = 'auto', delimiter: str = 'auto'):
  28. """Yield (logical data row, physical end line, headers, values); header row is 0."""
  29. csv.field_size_limit(MAX_FIELD_SIZE)
  30. if path.suffix.lower() in {'.xlsx', '.xlsm'}:
  31. from openpyxl import load_workbook
  32. workbook = load_workbook(path, read_only=True, data_only=True)
  33. try:
  34. headers = None
  35. logical = 0
  36. for line, row in enumerate(workbook.worksheets[0].iter_rows(values_only=True), 1):
  37. values = ['' if cell is None else str(cell) for cell in row]
  38. if headers is None:
  39. if not any(values):
  40. continue
  41. headers = values
  42. yield 0, line, headers, None
  43. elif any(values):
  44. logical += 1
  45. yield logical, line, headers, values
  46. if headers is None:
  47. raise ValueError('输入文件为空,没有表头')
  48. finally:
  49. workbook.close()
  50. return
  51. encoding = detect_encoding(path) if encoding == 'auto' else encoding
  52. with path.open(encoding=encoding, newline='') as handle:
  53. if delimiter == 'auto':
  54. position = handle.tell()
  55. first = next((line for line in handle if line.strip()), '')
  56. if not first:
  57. raise ValueError('输入文件为空,没有表头')
  58. # Header names contain no declaration values. Using the header avoids
  59. # mistaking G_MODEL's pipes or semicolons for file delimiters.
  60. delimiter = max((',', '\t', ';', '|'), key=lambda sep: len(next(csv.reader([first], delimiter=sep))))
  61. handle.seek(position)
  62. reader = csv.reader(handle, delimiter=delimiter, strict=True)
  63. headers = None
  64. logical = 0
  65. try:
  66. for values in reader:
  67. if not values or not any(value.strip() for value in values):
  68. continue
  69. if headers is None:
  70. headers = values
  71. yield 0, reader.line_num, headers, None
  72. else:
  73. logical += 1
  74. yield logical, reader.line_num, headers, values
  75. except csv.Error as error:
  76. raise ValueError(f'CSV 第 {reader.line_num} 行读取失败:{error};请检查引号或超长字段') from error
  77. if headers is None:
  78. raise ValueError('输入文件为空,没有表头')