structure.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278
  1. """Conservative structural reading of irregular declaration-element text."""
  2. import re
  3. from functools import lru_cache
  4. PRIMARY = set("||;;\t\r\n")
  5. QUOTES = {'"': '"', "'": "'", '“': '”', '‘': '’'}
  6. EMPTY_MARKERS = {'/', '/'}
  7. NO_BRAND = re.compile(r'^(?:无品牌|无牌|无厂牌|无商标|无品牌信息|无品牌标识|无品牌名称)$')
  8. BRAND_QUALIFIER = re.compile(r'^(?:无中文品牌|无外文品牌|无中文名称|无外文名称|无中文商标|无外文商标)$')
  9. # “无型号/无规格”这类显式缺项,属于“已经说清楚没有”,不算识别失败。
  10. NO_MODEL = re.compile(
  11. r'^(?:无型号|无中文型号|无外文型号|无型号信息|无具体型号|无型号标识|无牌号|无规格|无规格型号|'
  12. r'无货号|无零件号|未提供型号|未提供规格|不适用|无|—|–|-|/|/|N/?A|NA|NONE|NULL|空)$', re.I)
  13. # 标签:品牌/型号的常见写法都算线索,长写法放在前面避免被短写法截断。
  14. LABEL = re.compile(
  15. r'(?<!\w)(品牌名称|外文品牌|中文品牌|品牌商标|品牌|厂牌|商标|制造商|生产商|生产厂家|'
  16. r'型号规格|规格型号|产品型号|型号编号|型号|料号|物料编码|物料号|零件号|货号|规格|'
  17. r'用途|应用领域|应用场景|功能|'
  18. r'BRAND|MODEL|USAGE|FUNCTION)'
  19. r'\s*(?:[::=、]|为|是|["“”])\s*', re.I,
  20. )
  21. # 把同义标签归一到两个主类,后续只认“品牌/型号”。
  22. LABEL_ALIAS = {
  23. '品牌名称': '品牌', '外文品牌': '品牌', '中文品牌': '品牌', '品牌商标': '品牌',
  24. '厂牌': '品牌', '商标': '品牌', '制造商': '品牌', '生产商': '品牌', '生产厂家': '品牌',
  25. '型号规格': '型号', '规格型号': '型号', '产品型号': '型号', '型号编号': '型号',
  26. '料号': '型号', '物料编码': '型号', '物料号': '型号', '零件号': '型号', '货号': '型号',
  27. '规格': '型号',
  28. }
  29. # “规格”这类弱标签只在一个值明显像型号时才采用,避免把 0402、100nF 这类参数当成型号。
  30. WEAK_LABELS = {'规格'}
  31. # 括号里写标签的两种写法:“(品牌)MURATA” 与 “MURATA(品牌)”。
  32. PAREN_LABEL = re.compile(
  33. r'[((]\s*(品牌名称|外文品牌|中文品牌|品牌|厂牌|商标|制造商|生产商|'
  34. r'型号规格|规格型号|产品型号|型号|料号|物料编码|物料号|零件号|货号|规格)'
  35. r'\s*[))]', re.I)
  36. # 一个值前后残留的分隔符/引号/括号。
  37. TRIM_CHARS = ' \t\u3000_、,,||;;:://"\'“”‘’()()【】[]'
  38. # 值里混进了另一个标签,说明取值越界了,从这里截断。
  39. INNER_LABEL = re.compile(r'(?<!\w)(品牌名称|外文品牌|中文品牌|品牌|厂牌|商标|'
  40. r'型号规格|规格型号|产品型号|型号|料号|物料编码|物料号|零件号|货号|规格)')
  41. NUMBERED = re.compile(r'^\s*\d{1,2}\s*[、..]\s*')
  42. STATUS = re.compile(r'^[0-4]$')
  43. CHINESE_STATUS = re.compile(r'^[\u3400-\u9fff]{1,12}$')
  44. def unwrap_outer_quotes(value):
  45. text = str(value or '').strip()
  46. while len(text) >= 2 and text[0] in QUOTES and text[-1] == QUOTES[text[0]]:
  47. quote = text[0]
  48. text = text[1:-1].strip()
  49. if quote in ('"', "'"):
  50. text = text.replace(quote * 2, quote)
  51. return text
  52. def _primary_tokens(text):
  53. parts, buffer = [], []
  54. quote = None
  55. bracket = 0
  56. for char in text:
  57. if quote:
  58. buffer.append(char)
  59. if char == quote:
  60. quote = None
  61. continue
  62. if char in QUOTES:
  63. quote = QUOTES[char]
  64. buffer.append(char)
  65. elif char in '((':
  66. bracket += 1
  67. buffer.append(char)
  68. elif char in '))' and bracket:
  69. bracket -= 1
  70. buffer.append(char)
  71. elif char in PRIMARY and not bracket:
  72. parts.append(''.join(buffer).strip())
  73. buffer = []
  74. else:
  75. buffer.append(char)
  76. parts.append(''.join(buffer).strip())
  77. return parts
  78. def _slash_tokens(text):
  79. parts, buffer = [], []
  80. for index, char in enumerate(text):
  81. if char not in '//':
  82. buffer.append(char)
  83. continue
  84. before = text[index - 1] if index else ''
  85. after = text[index + 1] if index + 1 < len(text) else ''
  86. current = ''.join(buffer).strip()
  87. # The first status pair can use slash; later numeric fractions stay whole.
  88. first_status_pair = not parts and STATUS.fullmatch(current) and bool(after) and after in '01234'
  89. if before.isdigit() and after.isdigit() and current.isdigit() and not first_status_pair:
  90. buffer.append(char)
  91. else:
  92. parts.append(current)
  93. buffer = []
  94. parts.append(''.join(buffer).strip())
  95. return parts
  96. def _underscore_tokens(text):
  97. parts = [part.strip() for part in text.split('_')]
  98. return parts if any(parts) else [text]
  99. def _space_tokens(text):
  100. parts = [part for part in re.split(r'\s{1,}', text.strip()) if part]
  101. return parts
  102. def split_attributes(raw, is_brand=None):
  103. """Return candidate fields, split mode, and a structure warning if needed."""
  104. text = unwrap_outer_quotes(raw)
  105. parts = _primary_tokens(text)
  106. mode = 'primary'
  107. warning = ''
  108. labelled = bool(LABEL.search(NUMBERED.sub('', text)))
  109. if len(parts) == 1 and text.count('_') >= 2 and not labelled:
  110. proposed = _underscore_tokens(text)
  111. # 下划线在真实数据里常被当作字段分隔;只有在能得到多个非空段时才采用。
  112. if len(proposed) >= 3 and sum(bool(x) for x in proposed) >= 2:
  113. parts, mode, warning = proposed, 'underscore_inferred', '下划线分段待核查'
  114. if len(parts) == 1 and text.count('_') < 2 and not labelled:
  115. proposed = _space_tokens(text)
  116. # 空格只在明显是多字段时才切,避免拆散 "AP MEMORY" 这类品牌名。
  117. if len(proposed) >= 3:
  118. parts, mode, warning = proposed, 'space_inferred', '空格分段待核查'
  119. if len(parts) == 1 and text.count('/') + text.count('/') >= 2:
  120. proposed = _slash_tokens(text)
  121. first_two_status = (len(proposed) >= 3 and
  122. STATUS.fullmatch(proposed[0]) and STATUS.fullmatch(proposed[1]))
  123. first_two_chinese = (len(proposed) >= 5 and
  124. CHINESE_STATUS.fullmatch(proposed[0]) and
  125. CHINESE_STATUS.fullmatch(proposed[1]))
  126. # “境内自主品牌/2/…” 这类中文品牌类型 + 数字状态,也是可靠的结构线索。
  127. chinese_then_status = (len(proposed) >= 3 and
  128. CHINESE_STATUS.fullmatch(proposed[0]) and
  129. STATUS.fullmatch(proposed[1]))
  130. # 前几个字段里出现已知品牌,说明这确实是一条属性串而不是自由文本。
  131. early_brand = (is_brand is not None and
  132. any(is_brand(part) for part in proposed[1:5] if part))
  133. labelled = sum(bool(LABEL.search(NUMBERED.sub('', part))) for part in proposed) >= 2
  134. first_brand = (len(proposed) >= 3 and
  135. (NO_BRAND.fullmatch(proposed[0]) or
  136. (is_brand is not None and is_brand(proposed[0])) or
  137. (BRAND_QUALIFIER.fullmatch(proposed[0]) and
  138. is_brand is not None and is_brand(proposed[1]))))
  139. if first_two_status or first_two_chinese or chinese_then_status or early_brand or labelled or first_brand:
  140. parts, mode, warning = proposed, 'slash_inferred', '纯斜杠分段待核查'
  141. parts = [unwrap_outer_quotes(part) if part not in EMPTY_MARKERS else '' for part in parts]
  142. return parts, mode, warning
  143. def clean_value(value, separator=''):
  144. """把一个标签取值修干净:去掉两端残留的分隔符/引号,并在混入另一个标签处截断。"""
  145. text = str(value or '').strip(TRIM_CHARS)
  146. inner = INNER_LABEL.search(text)
  147. if inner:
  148. text = text[:inner.start()].strip(TRIM_CHARS)
  149. if separator == '、':
  150. # 用顿号做键值分隔的记录,值本身也在顿号处结束
  151. text = re.split(r'[、]', text, maxsplit=1)[0].strip(TRIM_CHARS)
  152. return text
  153. # 键和值被主分隔符切开、各自占一段的写法:品牌|MURATA|型号|X
  154. KEY_TOKEN = re.compile(
  155. r'^(?:品牌名称|外文品牌|中文品牌|品牌|厂牌|商标|制造商|生产商|生产厂家|'
  156. r'型号规格|规格型号|产品型号|型号|料号|物料编码|物料号|零件号|货号|BRAND|MODEL)$', re.I)
  157. # 走正则之前先做一次子串判断:不含任何标签词的段直接跳过,省掉三次正则扫描。
  158. # 只放"真出现就可能是标签"的词,像“型”“牌”这种到处都有的字不放,否则等于没筛。
  159. LABEL_HINTS = ('品牌', '型号', '厂牌', '商标', '制造', '生产', '料号', '物料', '零件', '货号',
  160. '规格', '用途', '应用', '功能')
  161. LATIN_HINTS = ('BRAND', 'MODEL', 'USAGE', 'FUNCTION')
  162. def label_kind(name):
  163. """同义标签归一:品牌系列 → 品牌,型号系列 → 型号,其余原样大写。"""
  164. mapped = LABEL_ALIAS.get(name, LABEL_ALIAS.get(str(name).upper(), name))
  165. return str(mapped).upper()
  166. def looks_like_model(text):
  167. """一段文字是否明显像料号:够长、有字母也有数字。
  168. 用来替代"靠位置猜型号",所以门槛保守——100nF、0402、DDR4、5G基站 都不算。
  169. """
  170. value = str(text or '')
  171. return (len(value) >= 6 and len(re.findall(r'[A-Za-z]', value)) >= 2
  172. and len(re.findall(r'\d', value)) >= 3)
  173. # 申报要素常把“这个品牌/型号以及同类”写成“XXX等牌”“XXX等型”,
  174. # 后缀不是名字的一部分,留着会把同一个品牌/型号劈成多条。
  175. BRAND_SUFFIXES = ('等品牌', '等商标', '等厂牌', '等厂商', '等牌', '等', '牌', '商标')
  176. MODEL_SUFFIXES = ('等型号', '等规格', '等牌号', '等型', '等')
  177. def strip_brand_suffix(text):
  178. value = str(text or '').strip()
  179. # “无品牌/无中文品牌”本身就是完整语义,不能把结尾的“牌”当后缀剥掉
  180. if NO_BRAND.fullmatch(value) or BRAND_QUALIFIER.fullmatch(value):
  181. return value, False
  182. for suffix in BRAND_SUFFIXES:
  183. if value.endswith(suffix) and len(value) > len(suffix):
  184. return value[:-len(suffix)].strip(), True
  185. return value, False
  186. def strip_model_suffix(text):
  187. value = str(text or '').strip()
  188. for suffix in MODEL_SUFFIXES:
  189. if value.endswith(suffix) and len(value) > len(suffix):
  190. return value[:-len(suffix)].strip(), True
  191. return value, False
  192. def labels_in(text):
  193. """Find explicit labels within one candidate field, preserving sentence values.
  194. 支持四种写法:
  195. · 键值带分隔符:品牌:MURATA / 品牌为MURATA / 型号"X" / 品牌、村田
  196. · 括号里放标签:(品牌)MURATA
  197. · 标签放在值后面:MURATA(品牌)
  198. 弱标签(如“规格”)只有在值看起来像型号时才采用,避免把 0402 当成型号。
  199. 先做一次廉价的子串判断再走正则,并对结果做缓存——解析 400 万条时这是最热的一段。
  200. """
  201. if not text:
  202. return iter(())
  203. probe = str(text)
  204. upper = probe.upper()
  205. if (not any(hint in probe for hint in LABEL_HINTS)
  206. and not any(hint in upper for hint in LATIN_HINTS)):
  207. return iter(())
  208. return iter(_labels_cached(probe))
  209. @lru_cache(maxsize=32768)
  210. def _labels_cached(text):
  211. return tuple(_labels_impl(text))
  212. def _labels_impl(text):
  213. text = NUMBERED.sub('', text)
  214. # “(品牌)MURATA” → “品牌:MURATA”,统一交给下面的标签正则处理
  215. normalised = PAREN_LABEL.sub(lambda m: f'{m.group(1)}:', text)
  216. matches = list(LABEL.finditer(normalised))
  217. for index, match in enumerate(matches):
  218. end = matches[index + 1].start() if index + 1 < len(matches) else len(normalised)
  219. separator = normalised[match.end(1):match.end()].strip()
  220. raw = normalised[match.end():end]
  221. value = clean_value(raw, '、' if '、' in separator else '')
  222. if not value:
  223. continue
  224. name = match.group(1)
  225. if name in WEAK_LABELS and not looks_like_model(value):
  226. continue
  227. yield label_kind(name), value
  228. # “MURATA(品牌)”:标签在值后面,取括号前面那一段
  229. for match in PAREN_LABEL.finditer(text):
  230. head = text[:match.start()]
  231. if PAREN_LABEL.search(head):
  232. # 前面已经出现过括号标签,说明这段是被前面的“(品牌)MURATA”形式吃掉的,跳过
  233. continue
  234. tail = re.split(r'[、,,||;;]', head)[-1]
  235. value = clean_value(tail)
  236. if value:
  237. name = match.group(1)
  238. if name in WEAK_LABELS and not looks_like_model(value):
  239. continue
  240. yield label_kind(name), value