"""Conservative structural reading of irregular declaration-element text.""" import re from functools import lru_cache PRIMARY = set("||;;\t\r\n") QUOTES = {'"': '"', "'": "'", '“': '”', '‘': '’'} EMPTY_MARKERS = {'/', '/'} NO_BRAND = re.compile(r'^(?:无品牌|无牌|无厂牌|无商标|无品牌信息|无品牌标识|无品牌名称)$') BRAND_QUALIFIER = re.compile(r'^(?:无中文品牌|无外文品牌|无中文名称|无外文名称|无中文商标|无外文商标)$') # “无型号/无规格”这类显式缺项,属于“已经说清楚没有”,不算识别失败。 NO_MODEL = re.compile( r'^(?:无型号|无中文型号|无外文型号|无型号信息|无具体型号|无型号标识|无牌号|无规格|无规格型号|' r'无货号|无零件号|未提供型号|未提供规格|不适用|无|—|–|-|/|/|N/?A|NA|NONE|NULL|空)$', re.I) # 标签:品牌/型号的常见写法都算线索,长写法放在前面避免被短写法截断。 LABEL = re.compile( r'(?= 2 and text[0] in QUOTES and text[-1] == QUOTES[text[0]]: quote = text[0] text = text[1:-1].strip() if quote in ('"', "'"): text = text.replace(quote * 2, quote) return text def _primary_tokens(text): parts, buffer = [], [] quote = None bracket = 0 for char in text: if quote: buffer.append(char) if char == quote: quote = None continue if char in QUOTES: quote = QUOTES[char] buffer.append(char) elif char in '((': bracket += 1 buffer.append(char) elif char in '))' and bracket: bracket -= 1 buffer.append(char) elif char in PRIMARY and not bracket: parts.append(''.join(buffer).strip()) buffer = [] else: buffer.append(char) parts.append(''.join(buffer).strip()) return parts def _slash_tokens(text): parts, buffer = [], [] for index, char in enumerate(text): if char not in '//': buffer.append(char) continue before = text[index - 1] if index else '' after = text[index + 1] if index + 1 < len(text) else '' current = ''.join(buffer).strip() # The first status pair can use slash; later numeric fractions stay whole. first_status_pair = not parts and STATUS.fullmatch(current) and bool(after) and after in '01234' if before.isdigit() and after.isdigit() and current.isdigit() and not first_status_pair: buffer.append(char) else: parts.append(current) buffer = [] parts.append(''.join(buffer).strip()) return parts def _underscore_tokens(text): parts = [part.strip() for part in text.split('_')] return parts if any(parts) else [text] def _space_tokens(text): parts = [part for part in re.split(r'\s{1,}', text.strip()) if part] return parts def split_attributes(raw, is_brand=None): """Return candidate fields, split mode, and a structure warning if needed.""" text = unwrap_outer_quotes(raw) parts = _primary_tokens(text) mode = 'primary' warning = '' labelled = bool(LABEL.search(NUMBERED.sub('', text))) if len(parts) == 1 and text.count('_') >= 2 and not labelled: proposed = _underscore_tokens(text) # 下划线在真实数据里常被当作字段分隔;只有在能得到多个非空段时才采用。 if len(proposed) >= 3 and sum(bool(x) for x in proposed) >= 2: parts, mode, warning = proposed, 'underscore_inferred', '下划线分段待核查' if len(parts) == 1 and text.count('_') < 2 and not labelled: proposed = _space_tokens(text) # 空格只在明显是多字段时才切,避免拆散 "AP MEMORY" 这类品牌名。 if len(proposed) >= 3: parts, mode, warning = proposed, 'space_inferred', '空格分段待核查' if len(parts) == 1 and text.count('/') + text.count('/') >= 2: proposed = _slash_tokens(text) first_two_status = (len(proposed) >= 3 and STATUS.fullmatch(proposed[0]) and STATUS.fullmatch(proposed[1])) first_two_chinese = (len(proposed) >= 5 and CHINESE_STATUS.fullmatch(proposed[0]) and CHINESE_STATUS.fullmatch(proposed[1])) # “境内自主品牌/2/…” 这类中文品牌类型 + 数字状态,也是可靠的结构线索。 chinese_then_status = (len(proposed) >= 3 and CHINESE_STATUS.fullmatch(proposed[0]) and STATUS.fullmatch(proposed[1])) # 前几个字段里出现已知品牌,说明这确实是一条属性串而不是自由文本。 early_brand = (is_brand is not None and any(is_brand(part) for part in proposed[1:5] if part)) labelled = sum(bool(LABEL.search(NUMBERED.sub('', part))) for part in proposed) >= 2 first_brand = (len(proposed) >= 3 and (NO_BRAND.fullmatch(proposed[0]) or (is_brand is not None and is_brand(proposed[0])) or (BRAND_QUALIFIER.fullmatch(proposed[0]) and is_brand is not None and is_brand(proposed[1])))) if first_two_status or first_two_chinese or chinese_then_status or early_brand or labelled or first_brand: parts, mode, warning = proposed, 'slash_inferred', '纯斜杠分段待核查' parts = [unwrap_outer_quotes(part) if part not in EMPTY_MARKERS else '' for part in parts] return parts, mode, warning def clean_value(value, separator=''): """把一个标签取值修干净:去掉两端残留的分隔符/引号,并在混入另一个标签处截断。""" text = str(value or '').strip(TRIM_CHARS) inner = INNER_LABEL.search(text) if inner: text = text[:inner.start()].strip(TRIM_CHARS) if separator == '、': # 用顿号做键值分隔的记录,值本身也在顿号处结束 text = re.split(r'[、]', text, maxsplit=1)[0].strip(TRIM_CHARS) return text # 键和值被主分隔符切开、各自占一段的写法:品牌|MURATA|型号|X KEY_TOKEN = re.compile( r'^(?:品牌名称|外文品牌|中文品牌|品牌|厂牌|商标|制造商|生产商|生产厂家|' r'型号规格|规格型号|产品型号|型号|料号|物料编码|物料号|零件号|货号|BRAND|MODEL)$', re.I) # 走正则之前先做一次子串判断:不含任何标签词的段直接跳过,省掉三次正则扫描。 # 只放"真出现就可能是标签"的词,像“型”“牌”这种到处都有的字不放,否则等于没筛。 LABEL_HINTS = ('品牌', '型号', '厂牌', '商标', '制造', '生产', '料号', '物料', '零件', '货号', '规格', '用途', '应用', '功能') LATIN_HINTS = ('BRAND', 'MODEL', 'USAGE', 'FUNCTION') def label_kind(name): """同义标签归一:品牌系列 → 品牌,型号系列 → 型号,其余原样大写。""" mapped = LABEL_ALIAS.get(name, LABEL_ALIAS.get(str(name).upper(), name)) return str(mapped).upper() def looks_like_model(text): """一段文字是否明显像料号:够长、有字母也有数字。 用来替代"靠位置猜型号",所以门槛保守——100nF、0402、DDR4、5G基站 都不算。 """ value = str(text or '') return (len(value) >= 6 and len(re.findall(r'[A-Za-z]', value)) >= 2 and len(re.findall(r'\d', value)) >= 3) # 申报要素常把“这个品牌/型号以及同类”写成“XXX等牌”“XXX等型”, # 后缀不是名字的一部分,留着会把同一个品牌/型号劈成多条。 BRAND_SUFFIXES = ('等品牌', '等商标', '等厂牌', '等厂商', '等牌', '等', '牌', '商标') MODEL_SUFFIXES = ('等型号', '等规格', '等牌号', '等型', '等') def strip_brand_suffix(text): value = str(text or '').strip() # “无品牌/无中文品牌”本身就是完整语义,不能把结尾的“牌”当后缀剥掉 if NO_BRAND.fullmatch(value) or BRAND_QUALIFIER.fullmatch(value): return value, False for suffix in BRAND_SUFFIXES: if value.endswith(suffix) and len(value) > len(suffix): return value[:-len(suffix)].strip(), True return value, False def strip_model_suffix(text): value = str(text or '').strip() for suffix in MODEL_SUFFIXES: if value.endswith(suffix) and len(value) > len(suffix): return value[:-len(suffix)].strip(), True return value, False def labels_in(text): """Find explicit labels within one candidate field, preserving sentence values. 支持四种写法: · 键值带分隔符:品牌:MURATA / 品牌为MURATA / 型号"X" / 品牌、村田 · 括号里放标签:(品牌)MURATA · 标签放在值后面:MURATA(品牌) 弱标签(如“规格”)只有在值看起来像型号时才采用,避免把 0402 当成型号。 先做一次廉价的子串判断再走正则,并对结果做缓存——解析 400 万条时这是最热的一段。 """ if not text: return iter(()) probe = str(text) upper = probe.upper() if (not any(hint in probe for hint in LABEL_HINTS) and not any(hint in upper for hint in LATIN_HINTS)): return iter(()) return iter(_labels_cached(probe)) @lru_cache(maxsize=32768) def _labels_cached(text): return tuple(_labels_impl(text)) def _labels_impl(text): text = NUMBERED.sub('', text) # “(品牌)MURATA” → “品牌:MURATA”,统一交给下面的标签正则处理 normalised = PAREN_LABEL.sub(lambda m: f'{m.group(1)}:', text) matches = list(LABEL.finditer(normalised)) for index, match in enumerate(matches): end = matches[index + 1].start() if index + 1 < len(matches) else len(normalised) separator = normalised[match.end(1):match.end()].strip() raw = normalised[match.end():end] value = clean_value(raw, '、' if '、' in separator else '') if not value: continue name = match.group(1) if name in WEAK_LABELS and not looks_like_model(value): continue yield label_kind(name), value # “MURATA(品牌)”:标签在值后面,取括号前面那一段 for match in PAREN_LABEL.finditer(text): head = text[:match.start()] if PAREN_LABEL.search(head): # 前面已经出现过括号标签,说明这段是被前面的“(品牌)MURATA”形式吃掉的,跳过 continue tail = re.split(r'[、,,||;;]', head)[-1] value = clean_value(tail) if value: name = match.group(1) if name in WEAK_LABELS and not looks_like_model(value): continue yield label_kind(name), value