"""Workbook rules, two-stage application matching and conservative G_MODEL parsing.""" import re import unicodedata from collections import Counter, defaultdict from functools import lru_cache from openpyxl import load_workbook from .structure import (BRAND_QUALIFIER, CHINESE_STATUS, KEY_TOKEN, NO_BRAND, NO_MODEL, clean_value, label_kind, labels_in, looks_like_model, split_attributes, strip_brand_suffix, strip_model_suffix) from .brandmatch import BrandMatcher MEMORY_HS = {'8542321010', '8542321090', '8542329010', '8542329090'} # 这些词在品牌位上只表示“没有/不适用”,不能当成真实品牌参与自动扫描。 PLACEHOLDER = {'无', '无品牌', '无牌', '不适用', '其他', '其它', 'NA', 'N/A', 'NONE', 'NIL', '待定', '未提供', '未知', '未识别', '-', '—', '/'} # 别名字符串里常见的说明性前缀,拆片段时要先去掉。 ALIAS_LABEL = re.compile( r'(?:中文名称|外文名称|中文名|外文名|英文名称|品牌名称|商品名称|品牌|品名|品|BRAND|MODEL|NAME)' r'\s*[::=]\s*', re.I) ALIAS_SPLIT = re.compile(r'[,,、//;;||\s]+') # 拆出来的片段若只是通用词或国别,作为品牌线索只会制造误命中,不建索引。 ALIAS_STOP = { '英文', '中文', '外文', '品牌', '品名', '名称', '商品名称', '型号', '等', '无中文名称', '无外文名称', '无中文品牌', '无外文品牌', '无中文名', '无外文名', '无品牌', '无牌', '其他', '其它', '未知', '不适用', '不详', '不祥', '株式会社', '有限公司', '公司', '集团', '电子', '电气', '半导体', '科技', '芯片', '中国', '日本', '韩国', '美国', '德国', '法国', '英国', '荷兰', '瑞士', '奥地利', '新加坡', '马来西亚', '泰国', '越南', '印度', '台湾', '中国台湾', '香港', '深圳', } def alias_tokens(raw): """把一个申报写法拆成可索引的片段:整串、去标签、括号内外、按分隔符切开的各项。 真实申报里经常写成“品:KOA等 中文名称:兴亚等”或“KOA/兴亚”“KOA,日本兴亚”, 只按整串建索引时,“品牌:日本兴亚”这类写法永远匹配不上;拆出片段后即可命中。 """ text = unicodedata.normalize('NFKC', str(raw or '')).strip() out = [] def add(value): value = str(value or '').strip().strip('"\'“”‘’ \t,,;;/.') if value.endswith('等') and len(value) > 1: value = value[:-1] if value and value not in out: out.append(value) add(text) body = ALIAS_LABEL.sub(' ', text) add(body) for inner in re.findall(r'[((]([^))]*)[))]', body): add(inner) for part in ALIAS_SPLIT.split(re.sub(r'[((][^))]*[))]', ' ', body)): add(part) return out def norm(value): return re.sub(r'\s+', '', unicodedata.normalize('NFKC', str(value or ''))).upper() # 纯电气参数:容值、耐压、功率、尺寸、频率,这些不是型号。 SPEC_LIKE = re.compile( r'^(?:\d+(?:\.\d+)?\s*(?:[nuµμpP]?F|[VvWwAa]|Hz|KHz|MHz|%|Ω|ohm)|\d{4}|\d+(?:\.\d+)?)$') # 像品牌名的写法(纯拉丁,不含中文),用来提示“映射表里可能缺这个品牌”。 LATIN_ONLY = re.compile(r'[A-Za-z][A-Za-z0-9 .&+\-/]{1,30}') # 规则表里的兜底标签:不是真品牌,模糊/拼写/包含命中到它们一律不认。 CATCH_ALL = {'Others', '其他', '其它', '其他品牌', '未识别', '未知', 'UNKNOWN'} # 明显是“品牌字段”的写法(带 牌/品牌 字样),用于在品牌认不出时仍能取到后面的型号。 BRAND_FIELD = re.compile(r'(?:牌|品牌|商标)\s*$') def model_candidate(text, is_brand=None, app_signal=None): """判断一段文字能否当型号用:不是参数、不是别的标签的值、有字母也有数字。""" value = clean_value(text) if not value or SPEC_LIKE.match(value): return '' labels = list(labels_in(value)) if labels and not any(kind in ('品牌', 'BRAND', '型号', 'MODEL') for kind, _ in labels): return '' if not (re.search(r'[A-Za-z]', value) and re.search(r'\d', value)): return '' if is_brand and is_brand(value): return '' if app_signal and app_signal(value): return '' return value def canon_label(value): """品牌标签的归并键:去掉括号说明与空白后大写。""" text = unicodedata.normalize('NFKC', str(value or '')).strip() text = re.sub(r'[((][^))]*[))]', '', text).strip() return re.sub(r'\s+', '', text).upper() def unique(values): values = set(values) return next(iter(values)) if len(values) == 1 else '' class Rules: def __init__(self, mapping, fields, country_system='ISO', fuzzy_contains=True): self.alias, self.brand_country, self.models = defaultdict(set), defaultdict(set), defaultdict(set) self.prefixes, self.first, self.second, self.cap, self.schema = [], {}, [], {}, {} self.conflicts = [] self.fuzzy_contains = fuzzy_contains w = load_workbook(mapping, read_only=True, data_only=True) pairs = [(str(b).strip(), str(r).strip()) for b, r in w['品牌名称映射规则'].iter_rows(min_row=2, values_only=True) if b and r] # 品牌标签存在 “MICRON/Micron”“AMD/AMD(XILINX)” 等同一品牌的多种写法, # 会让同一申报内容映射到多个标签而被整条判为冲突。先归并再建别名表。 counts = Counter(label for label, _ in pairs) display = {} for label in counts: key = canon_label(label) current = display.get(key) if current is None or ((counts[label], -len(label), label) > (counts[current], -len(current), current)): display[key] = label self.label_map = {label: display[canon_label(label)] for label in counts} self.label_merges = sorted((a, b) for a, b in self.label_map.items() if a != b) full = defaultdict(set) for brand, raw in pairs: shown = self.label_map[brand] full[norm(raw)].add(shown) full[norm(brand)].add(shown) # 别名字符串常带说明前缀或并列写法。整串之外再按片段建索引, # “品牌:日本兴亚”这类只含品牌一部分的写法才能命中兴亚(KOA)。 extra = defaultdict(set) for brand, raw in pairs: shown = self.label_map[brand] for token in alias_tokens(raw): key = norm(token) if not key or key in full or key in ALIAS_STOP: continue if len(key) < (2 if re.search(r'[\u3400-\u9fff]', key) else 3): continue extra[key].add(shown) for key, labels in extra.items(): # 片段若指向多个品牌(说明是“电子”“英文”这类通用词),保留只会制造冲突。 if len(labels) == 1: full[key] = labels self.alias = full # 记录“由片段推断出来”的键:型号表里的品牌中文名是显式品牌名,可以覆盖这类推断键。 self.fragment_keys = {key for key, labels in extra.items() if len(labels) == 1} for brand, country in w['品牌国别映射规则'].iter_rows(min_row=2, values_only=True): if brand and country: label = str(brand).strip() self.brand_country[norm(self.label_map.get(label, label))].add(str(country).strip()) for r in w['存储器型号分类映射规则'].iter_rows(min_row=2, values_only=True): en, cn, model, prefix, desc, c1, c2, c3, c4 = r if not model: continue brand = unique(self.alias[norm(en)]) or str(en or cn or '') for alias in (en, cn): if alias: # 型号表只补空位,或覆盖“由别名片段推断出来”的键(如 超微→SUPERMICRO); # 名称表里作为整串别名出现过的键不改写,避免同一写法被两个品牌认领。 key = norm(alias) current = self.alias.get(key) if key in self.fragment_keys: self.alias[key] = {brand} elif not current or brand in current: self.alias[key].add(brand) record = (brand, str(desc or ''), str(c2 or ''), str(c3 or ''), str(c4 or ''),str(c1 or '')) self.models[norm(model)].add(record) if prefix: self.prefixes.append((norm(prefix), record)) self.prefixes = sorted(set(self.prefixes), key=lambda x: -len(x[0])) self.prefix_lookup=defaultdict(set) for prefix,record in self.prefixes:self.prefix_lookup[prefix].add(record) # 四字片段索引:从任意一段文本里反查“里面藏着哪个已知型号”, # 用于“型号:K4B4G1646E等”“K4B4G1646E-BYK0”这类写法,避免全表逐条扫描。 self.model_grams = defaultdict(set) for key in self.models: for i in range(len(key) - 3): self.model_grams[key[i:i + 4]].add(key) for r in w['应用领域映射规则(一轮匹配)'].iter_rows(min_row=2, values_only=True): if r[0] and r[1]: self.first.setdefault(norm(r[0]), set()).add(str(r[1])) for r in w['应用领域与功能映射规则(二轮匹配)'].iter_rows(min_row=2, values_only=True): if r[0] not in ('其他', '通用领域') and r[1]: self.second.append((r[0], [norm(k) for k in str(r[1]).split('、')])) for r in w['电容器映射规则'].iter_rows(min_row=2, values_only=True): if r[2]: self.cap[str(r[2])[:10]] = str(r[1]) w.close() for kind, mapping in [('品牌', self.alias), ('品牌国别', self.brand_country), ('型号', self.models), ('应用一轮', self.first)]: for key, values in mapping.items(): if len(values) > 1: self.conflicts.append((kind, key, repr(sorted(values)))) for original, merged in self.label_merges: self.conflicts.append(('品牌标签归并', original, f'归并为 {merged}')) self.brand_matcher = BrandMatcher(self.alias, PLACEHOLDER | { '无中文品牌', '无外文品牌', '无中文名称', '无外文名称'}, allow_contains=fuzzy_contains) self._build_reverse_index() w = load_workbook(fields, read_only=True, data_only=True) for r in w['字段规则_申报要素'].iter_rows(min_row=3, values_only=True): if r[0]: self.schema[str(r[0])] = [str(x or '') for x in r[2:]] country_candidates=defaultdict(set) self.countries, self.ports = {}, {} for r in w['映射规则_国别地区代码'].iter_rows(min_row=2, values_only=True): if not r[6]: continue keys = [r[0]] if country_system == 'GEC' else [r[1], r[2], r[3]] for key in keys + [r[6]]: if key is not None: if str(key) in ['不适用','无']:continue country_candidates[norm(key)].add(str(r[6])) if str(key).isdigit(): country_candidates[str(key).zfill(3)].add(str(r[6])) for key,values in country_candidates.items(): if len(values)==1:self.countries[key]=next(iter(values)) else: self.countries[key]='映射冲突:'+key self.conflicts.append(('国家地区',key,repr(sorted(values)))) for code, cn, en, country in w['映射规则_港口代码'].iter_rows(min_row=2, values_only=True): if code: self.ports[norm(code)] = (str(cn or en or code), str(country or '未识别')) w.close() def country(self, raw): return self.countries.get(norm(raw), f'未映射:{raw}' if raw else '未识别') def port(self, raw): key = norm(raw) return self.ports.get(key, (f'未映射:{raw}', '未识别')) if key else ('未识别', '未识别') # ---------- 品牌 / 型号取词 ---------- @staticmethod def _clean(raw): text = unicodedata.normalize('NFKC', str(raw or '')).strip() return text.strip('"\'“”‘’ \t').strip() def brand_variants(self, raw): """按可靠性从高到低给出品牌候选写法:整体、去“牌”、括号内外、枚举各项。""" text = self._clean(raw) text = re.sub(r'^(?:品牌|BRAND)\s*[::=]\s*', '', text, flags=re.I) out = [] def add(value): value = str(value or '').strip().strip('"\'“”‘’ \t').strip() if value and value not in out: out.append(value) add(text) if text.endswith('牌') and len(text) > 1: add(text[:-1]) match = re.match(r'^(.*?)[((]([^))]*)[))](.*)$', text) if match: add(match.group(1) + match.group(3)) add(match.group(1)) add(match.group(2)) if match.group(2).endswith('牌') and len(match.group(2)) > 1: add(match.group(2)[:-1]) flat = re.sub(r'[((][^))]*[))]', ' ', text) for part in re.split(r'[,,、//;;||]+', flat): add(part) for part in re.split(r'[\s_]+', text): add(part) return out def _build_reverse_index(self): """为"规则表里的长申报内容包含 G_MODEL 写法"建索引。 品牌表里有很多长串,比如 `PANJIT牌/强茂牌`、`村田制作所(MURATA)`、 `NEXPERIA、DIODES、PANJIT/无中文名称`。反过来查"这段文字出现在哪条规则里", 用 3 字片倒排 + 逐条验证,避免每次全表扫描。 """ self._rev_keys = [] # [(规范化的键, 该键对应的品牌集合)] self._rev_grams = defaultdict(set) # 二字片 -> 键序号集合(中文两字就是完整品牌) for key, labels in self.alias.items(): packed = norm(key) if len(packed) < 2: continue index = len(self._rev_keys) self._rev_keys.append((packed, frozenset(labels))) for gram in {packed[i:i + 2] for i in range(len(packed) - 1)}: self._rev_grams[gram].add(index) @lru_cache(maxsize=16384) def reverse_brand_labels(self, text): """返回"规则表里的哪条申报内容包含这段文字"所对应的品牌。 例:'PANJIT' 会被 `PANJIT牌/强茂牌` 命中(长申报内容里含英文名)。 只认"除这段文字外没有别的已知品牌"的那几条规则——像 `NEXPERIA、DIODES、PANJIT/无中文名称` 这种多品牌混写不算, 否则会把 PANJIT 认成 Nexperia。 """ packed = norm(text) # 中文两字就可能是完整品牌(村田/强茂),拉丁字母要四个以上才不至于乱撞(ST/AP 不算) minimum = 2 if re.search(r'[\u4e00-\u9fff]', packed) else 4 if len(packed) < minimum: return frozenset() candidates = None for gram in {packed[i:i + 2] for i in range(len(packed) - 1)}: bucket = self._rev_grams.get(gram) if not bucket: return frozenset() candidates = set(bucket) if candidates is None else candidates & bucket if not candidates: return frozenset() if not candidates: return frozenset() labels = set() for index in candidates: packed_key, key_labels = self._rev_keys[index] if packed not in packed_key: continue if key_labels <= CATCH_ALL: continue # 兜底标签不算证据 # 只认"这条规则差不多就是这个品牌名"的行:留 8 个字符的余量给 牌/无中文名称 这类后缀。 # 像 COMCHIPONPANJITPYSEMTECH…(多个品牌连写)这种长串一律不认。 if len(packed_key) > len(packed) + 8: continue if not self._single_brand_rule(packed_key, packed): continue labels |= key_labels return frozenset(labels) def _single_brand_rule(self, packed_key, packed): """这条规则除了这段文字,还有没有写着别的已知品牌。""" rest = packed_key.replace(packed, '', 1) for piece in re.split(r'[、,,//;;||()()\s::]+', rest): piece = piece.strip() if len(piece) < 2 or re.fullmatch(r'[\d.\-]+', piece): continue found = self.alias.get(piece) or self.alias.get(piece.rstrip('牌')) if found: return False return True def brand_detail(self, raw, allow_typo=True): """Return labels, matched key, warning, method, score, ranked candidates.""" if norm(raw) in PLACEHOLDER or BRAND_QUALIFIER.fullmatch(str(raw or '')): return set(), '', '', '', 0.0, () variants = self.brand_variants(raw) whole = self.alias.get(norm(variants[0])) if variants else None if whole: return (set(whole) if len(whole) == 1 else set(), variants[0], '' if len(whole) == 1 else '品牌映射冲突', '整体精确', 100.0, tuple((v, 100.0, variants[0], '整体精确') for v in sorted(whole))) hits = [] hit = '' for variant in variants: if norm(variant) in PLACEHOLDER or BRAND_QUALIFIER.fullmatch(variant): continue found = self.alias.get(norm(variant)) if found: hits.append(set(found)) if not hit: hit = variant if hits: agreed = set(hits[0]).intersection(*hits[1:]) candidates = tuple((v, 100.0, hit, '取词精确') for v in sorted(set.union(*hits))) return (agreed if len(agreed) == 1 else set(), hit, '' if len(agreed) == 1 else '品牌映射冲突', '取词精确', 100.0, candidates) labels, hit, method, score, candidates = self.brand_matcher.match(tuple(variants), allow_typo) catch_all_hit = bool(labels) and set(labels) <= CATCH_ALL if labels and catch_all_hit and method in ('拼写近似', '包含匹配', '子串包含', '标点归一'): # 拼错一个字母就落到 Others/其他 这种兜底标签上,几乎一定是认错了 # (实测 PANJIT 被拼写近似到表里的错行 PANJJIT → Others)。宁可继续往下找。 labels = set() if not labels: # 反向包含:规则表里的申报内容是一长串(可能中英混写),只要这段文字 # 整段出现在里面就算命中。例:'PANJIT' ⊂ 'PANJIT牌/强茂牌'。 reversed_labels = None for variant in variants: text = norm(variant) # 太短的拉丁串(ST、AP)先不看;中文两字可能是完整品牌 minimum = 2 if re.search(r'[\u4e00-\u9fff]', text) else 4 if len(text) < minimum or re.search(r'\d', text): continue found = self.reverse_brand_labels(text) if not found: continue reversed_labels = set(found) if reversed_labels is None else reversed_labels & set(found) if reversed_labels and len(reversed_labels) == 1: return (reversed_labels, hit, '', '反向包含', 90.0, tuple((v, 90.0, hit, '反向包含') for v in sorted(reversed_labels))) if reversed_labels and len(reversed_labels) > 1: return set(), hit, '品牌反向包含冲突', '反向包含', 90.0, () if catch_all_hit: return set(), hit, '品牌近似命中兜底标签,已忽略', method, score, candidates return (set(labels) if len(labels) == 1 else set(), hit, '品牌映射冲突' if len(labels) > 1 else '', method, score, candidates) def resolve_brand(self, raw): return self.brand_detail(raw)[:3] def is_brand(self, raw): # Avoid spelling guesses on unlabelled model IDs containing digits. labels, _, note, *_ = self.brand_detail(raw, allow_typo=not bool(re.search(r'\d', str(raw)))) return bool(labels and not note) def model_inside(self, tokens, brand): """在整段文本里找回已知型号:文本包含某个已知型号即算命中,最长优先。 典型场景是“型号:K4B4G1646E等”“K4B4G1646E-BYK0”“K4B4G1646E K4A8G165WB”, 原逻辑只做整串/去“等”/空格拆分/前缀,这类型号会整条判为未识别。 """ best = None for token in tokens: text = norm(token) if len(text) < 4: continue candidates = set() for i in range(len(text) - 3): candidates |= self.model_grams.get(text[i:i + 4], set()) for key in candidates: if key not in text: continue records = {r for r in self.models[key] if not brand or norm(r[0]) == norm(brand)} if records and (best is None or len(key) > len(best[0])): best = (key, records) return best def model_variants(self, raw): """型号候选写法:整体、括号内外、按空白/下划线、按逗号顿号枚举。""" text = self._clean(raw) text = re.sub(r'^(?:型号|MODEL)\s*[::=]\s*', '', text, flags=re.I) out = [] def add(value): value = str(value or '').strip().strip('"\'“”‘’ \t').strip() if value and value not in out: out.append(value) add(text) match = re.match(r'^(.*?)[((]([^))]*)[))](.*)$', text) if match: add(match.group(1)) add(match.group(2)) add(match.group(1) + match.group(3)) for part in re.split(r'[\s_]+', text): add(part) for part in re.split(r'[,,、]+', text): add(part) return out def app_signal(self, token): """该字段文本能否支撑一个具体应用领域(“其他/未识别”不算信号)。""" text = norm(str(token or '').replace('_', ' ')) if not text or text in PLACEHOLDER: return False label = unique(self.first.get(text, ())) if label and label not in ('其他', '未识别', '通用领域'): return True return any(key in text for _, keys in self.second for key in keys) def hits_vocabulary(self, token): """该字段文本是否落在应用领域的已知词表里。""" text = norm(str(token or '').replace('_', ' ')) if not text or text in PLACEHOLDER: return False if text in self.first: return True return any(key in text for _, keys in self.second for key in keys) def application(self, texts): text = norm(' '.join(texts).replace('_', ' ')) if not text: return '未识别', '未识别' first = unique(self.first.get(text, ())) if not first: hits = set() for part in texts: hits.update(self.first.get(norm(part.replace('_', ' ')), ())) first = unique(hits) or ('通用领域' if hits else '其他') hits = set() for category, keys in self.second: candidate = text # Exclusions are phrase-level: satellite communication is high-tech; # computer systems are advanced computing, ordinary computers consumer. if category == '网络通信': candidate = candidate.replace('卫星通信', '') if category == '消费电子': candidate = candidate.replace('计算机系统', '') if category == '高新电子': candidate = candidate.replace('无人飞机', '') if any((bool(re.search(r'(? 1 else unique(hits) or '其他') @lru_cache(maxsize=8192) def parse(self, hs, raw): flags, brand_raw, model_raw, app = [], '', '', [] label_brand_fallback = '' tokens, split_mode, split_warning = split_attributes( raw, is_brand=self.is_brand) if split_mode in ('space_inferred', 'underscore_inferred'): labels, _, note, kind, score, _ = self.brand_detail(raw, allow_typo=False) if labels and not note and kind == '标点归一' and score == 100: tokens, split_mode, split_warning = [str(raw)], 'primary', '' if split_warning: flags.append(split_warning) if re.search(r'(?:^|[||;;])\s*[//]\s*(?:$|[||;;])', str(raw or '')): flags.append('斜杠占位字段') schema = self.schema.get(hs, []) expected = len([x for x in schema if x]) brand_slot = next((i for i, label in enumerate(schema) if label.startswith('品牌') and '类型' not in label), None) model_slot = next((i for i, label in enumerate(schema) if label == '型号'), None) first_is_status = bool(tokens and re.fullmatch('[0-4]', tokens[0])) chinese_status = bool(tokens and CHINESE_STATUS.fullmatch(tokens[0]) and brand_slot is not None and brand_slot < len(tokens) and self.is_brand(tokens[brand_slot])) extra_only_empty = len(tokens) > expected and not any(tokens[expected:]) short_but_anchored = (len(tokens) < expected and brand_slot is not None and model_slot is not None and model_slot < len(tokens) and self.is_brand(tokens[brand_slot])) # Accept missing optional tail only when the brand position is independently anchored. aligned = len(tokens) == expected or extra_only_empty or short_but_anchored complete = bool(schema and aligned and (first_is_status or chinese_status)) if complete: if extra_only_empty: flags.append('模板末尾空段') if short_but_anchored: flags.append('模板末尾字段缺失待核查') for label, token in zip(schema, tokens): if label.startswith('品牌') and '类型' not in label: brand_raw = token if label == '型号': model_raw = token if any(k in label for k in ['用途', '应用', '功能']): app.append(token) method = '顺序提取' # 只有"其它位置确实有真实品牌"才否定位置提取;“无/不适用”这类占位词不算线索, # 否则品牌位留空、GTIN/CAS 写“无”的记录会把已经取到的型号也一起丢掉。 other_brand = any(self.is_brand(t) and norm(t) not in PLACEHOLDER for t in tokens) if brand_raw and not self.is_brand(brand_raw) and other_brand: brand_raw,model_raw,app='','',[] method='缺项识别' flags.append('顺序位置与品牌线索冲突,转缺项识别') else: method = '缺项识别' # Labels take precedence; numbered prefixes such as "1、型号:" are accepted. label_brands, label_models = [], [] for index, token in enumerate(tokens): for label, value in labels_in(token): if label in ('品牌', 'BRAND'): label_brands.append(value) elif label in ('型号', 'MODEL'): label_models.append(value) else: app.append(value) # 键和值各占一段:品牌|MURATA|型号|GRM188… if KEY_TOKEN.fullmatch(token.strip()): nxt = tokens[index + 1] if index + 1 < len(tokens) else '' value = clean_value(nxt) if nxt and not KEY_TOKEN.fullmatch(nxt.strip()) else '' kind = label_kind(token.strip()) if value and kind == '品牌': label_brands.append(value) elif value and kind == '型号': label_models.append(value) brand_label_conflict = len({norm(value) for value in label_brands}) > 1 model_label_conflict = len({norm(value) for value in label_models}) > 1 if brand_label_conflict: brand_raw = '' flags.append('品牌标签冲突') elif label_brands: brand_raw = label_brands[0] # 标签里给了品牌:能匹配上就锁死,后面的扫描不再覆盖。 if NO_BRAND.fullmatch(brand_raw) or BRAND_QUALIFIER.fullmatch(brand_raw): flags.append('无品牌标记') else: probe = self.brand_detail(brand_raw) if probe[0] and not probe[2]: flags.append('品牌标签锁定') elif probe[2]: # 一句话里写了两个不同品牌:保留原文,交给后面统一判冲突,不擅自取一个。 pass else: head = re.split(r'[,,;;。]', brand_raw)[0].strip() if head and head != brand_raw: head_probe = self.brand_detail(head) if head_probe[0] and not head_probe[2]: # “品牌:TDK,用于手机”这种句子值:逗号后面是说明,取前半段 brand_raw = head flags.append('品牌标签锁定') probe = head_probe else: probe = () if not probe: flags.append('品牌标签未命中') label_brand_fallback, brand_raw = brand_raw, '' if model_label_conflict: # 一条记录里同时写了“型号:”“规格型号:”等,按“多型号取首个”处理,不整条丢弃。 model_raw = label_models[0] flags.append('多型号取首个') elif label_models: model_raw = label_models[0] # 明确写“无型号/无规格”的,视为已经说清楚,不再去别处凑一个型号,也不算识别失败。 no_model_marker = bool(model_raw and NO_MODEL.fullmatch(model_raw)) if no_model_marker: model_raw = '' flags.append('无型号标记') if not brand_raw and not brand_label_conflict: brand_raw = next((t for t in tokens if NO_BRAND.fullmatch(t)), '') if not brand_raw and not brand_label_conflict: # 逐字段按“整体 -> 去牌 -> 括号内外 -> 枚举项”解析,排除“无/不适用”等占位词。 found = [] for index, token in enumerate(tokens): if not token or norm(token) in PLACEHOLDER or BRAND_QUALIFIER.fullmatch(token) \ or re.fullmatch(r'[\d.]+', token): continue # Explicit model/application fields cannot supply a brand by fuzzy scanning. token_labels = list(labels_in(token)) if token_labels and not any(label in ('品牌', 'BRAND') for label, _ in token_labels): continue allow_typo = (index == brand_slot or (not re.search(r'\d', token) and not self.hits_vocabulary(token))) labels, hit, note, kind, score, candidates = self.brand_detail(token, allow_typo=allow_typo) if labels and not note: found.append((score, index, token, labels)) if found: best = max(row[0] for row in found) top = [row for row in found if row[0] == best] distinct = set.union(*(row[3] for row in top)) if len(distinct) == 1: brand_raw = top[0][2] else: flags.append('品牌位置不确定') if not brand_raw and label_brand_fallback: # 标签里写过、但匹配不上,且别处也没找到:保留原文,便于核查到底是哪种写法没覆盖。 brand_raw = label_brand_fallback # “ABACUS等牌”里的“等牌”不是名字的一部分,去掉再查表,否则同品牌会被劈成两条。 brand_raw, brand_had_deng = strip_brand_suffix(brand_raw) if brand_had_deng: flags.append('品牌带“等”后缀') brand_match_method, brand_match_score, brand_match_key, brand_candidates = '', 0.0, '', () brand_set = set() if not NO_BRAND.fullmatch(brand_raw): brand_set, brand_hit, brand_note, brand_match_method, brand_match_score, brand_candidates = self.brand_detail(brand_raw) brand_match_key = brand_hit if brand_note: flags.append(brand_note) elif brand_match_method in ('标点归一', '包含匹配', '拼写近似'): flags.append('品牌模糊匹配') elif brand_match_method == '子串包含': flags.append('品牌包含匹配') elif brand_hit and brand_hit != brand_raw: flags.append('品牌括号或枚举取词') brand = unique(brand_set) if NO_BRAND.fullmatch(brand_raw) or BRAND_QUALIFIER.fullmatch(brand_raw): flags.append('无品牌标记') if not model_raw and not model_label_conflict and not no_model_marker: found = [(i, t) for i, t in enumerate(tokens) if norm(t) in self.models] if found: model_raw = found[0][1] if len(found) > 1: flags.append('多型号取首个') if not model_raw and brand_raw and not model_label_conflict and not no_model_marker: for i, token in enumerate(tokens): # 带用途/应用等标签的段不能供型号:否则“用途:5G基站”会被当成型号。 token_labels = list(labels_in(token)) if token_labels and not any(label in ('品牌', 'BRAND', '型号', 'MODEL') for label, _ in token_labels): continue variants = self.brand_variants(token) hit = next((v for v in variants if norm(v) == norm(brand_raw)), '') if brand_match_key and brand_match_key in token and self.resolve_brand(token)[0] == {brand}: hit = brand_match_key if not hit and brand and self.resolve_brand(token)[0] == {brand}: hit = brand_match_key if brand_match_key in token else token if not hit: continue tail = clean_value(token[token.find(hit) + len(hit):]) if hit in token else '' following = tokens[i + 1] if i + 1 < len(tokens) else '' for candidate in (tail, following): value = model_candidate(candidate, self.is_brand, self.app_signal) if value: model_raw = value flags.append('品牌后型号推断') break break if not model_raw and not model_label_conflict and not no_model_marker and brand: # 型号写在品牌前面("GRM188R71H104KA93D MURATA"):只认同品牌紧挨着的那一段, # 且必须明显像料号,避免把 100nF、0402、5G基站 这类参数或用途当成型号。 anchor = next((i for i, t in enumerate(tokens) if norm(t) == norm(brand_raw)), None) if anchor is None and brand_match_key: anchor = next((i for i, t in enumerate(tokens) if brand_match_key in t), None) if anchor: cand = clean_value(tokens[anchor - 1]) if anchor > 0 else '' if (looks_like_model(cand) and cand != brand_raw and not self.is_brand(cand) and not self.app_signal(cand) and norm(cand) not in PLACEHOLDER): model_raw = cand flags.append('品牌前型号推断') if not model_raw and not model_label_conflict and not no_model_marker and brand: # 型号和品牌挤在同一段里("GRM188… MURATA"):把每段再拆成片段,挑明显像料号的那一个。 for token in tokens: for part in re.split(r'[\s_\u3000、,,()()]+', token): cand = clean_value(part) if (looks_like_model(cand) and not self.is_brand(cand) and not self.app_signal(cand) and norm(cand) not in PLACEHOLDER and norm(cand) != norm(brand_raw)): model_raw = cand flags.append('型号片段提取') break if model_raw: break # “A5E30295495R2W等型”里的“等型”不是型号的一部分:去掉,并按“多型号取首个”记账。 if not model_raw and not no_model_marker and not model_label_conflict: # 品牌认不出来也不能把型号一起丢掉:如果前一段明显是品牌字段(带“牌/品牌”), # 紧跟其后的那一段又像料号,就照取(型号本身不依赖品牌表)。 for index, token in enumerate(tokens[:-1]): if BRAND_FIELD.search(clean_value(token) or token.strip()): nxt = tokens[index + 1] value = clean_value(nxt) if (looks_like_model(value) and value != brand_raw and not self.is_brand(value) and not self.app_signal(value)): model_raw = value flags.append('品牌字段后型号推断') break # “A5E30295495R2W等型”里的“等型”不是型号的一部分:去掉,并按“多型号取首个”记账。 model_raw, model_had_deng = strip_model_suffix(model_raw) if model_had_deng: flags.append('多型号取首个') model = norm(model_raw) if model and model not in self.models: if model.endswith('等'): model = model[:-1] flags.append('多型号取首个') if any(mark in model_raw for mark in ',,、'): first = norm(re.split(r'[,,、]+', model_raw, maxsplit=1)[0]) if first: model = first flags.append('多型号取首个') elif re.search(r'[\s_]', model_raw) and model not in self.models: words = [norm(part) for part in re.split(r'[\s_]+', model_raw) if part] matches_in_words = [part for part in words if part in self.models] if len(matches_in_words) == 1: model = matches_in_words[0] flags.append('空格分隔型号推断') if not model and hs in MEMORY_HS and not model_label_conflict and not no_model_marker: # 型号可能和品牌挤在同一段(如“长江存储(YMTC) YMTC-XX”),逐字段找回。 for token in tokens: for variant in self.model_variants(token): key = norm(variant) if not key or key in PLACEHOLDER or key not in self.models: continue hits = self.models[key] if brand: hits = {r for r in hits if norm(r[0]) == norm(brand)} if len(hits) == 1: model_raw, model = variant, key flags.append('型号候选扫描') break if model: break matches = self.models.get(model, set()) if brand: matches = {r for r in matches if norm(r[0]) == norm(brand)} if not matches and model and hs in MEMORY_HS: for length in range(len(model),0,-1): hits={r for r in self.prefix_lookup.get(model[:length],()) if not brand or norm(r[0])==norm(brand)} if hits: matches=hits;flags.append('型号前缀匹配');break # 最后兜底:文本里含某个已知型号就算命中(“xxx等”“xxx-yyy”)。 # 只在型号表覆盖的 HS 上做,避免把无关商品的编码当型号。 if not matches and hs in MEMORY_HS and self.fuzzy_contains: hit = self.model_inside(tokens, brand) if hit: model, matches = hit flags.append('型号包含匹配') # Different descriptions that imply the same taxonomy are still a mapping conflict. mapped = next(iter(matches)) if len(matches) == 1 else None if len(matches) > 1: flags.append('型号映射冲突') if mapped and not brand and not any('品牌' in f and ('冲突' in f or '不确定' in f) for f in flags): brand = mapped[0] brand_match_method, brand_match_score = '型号反推', 0.0 if not brand and not NO_BRAND.fullmatch(brand_raw) and not BRAND_QUALIFIER.fullmatch(brand_raw): flags.append('品牌未识别') # 字段里有像品牌名的拉丁写法、但映射表查不到(如 ABACUS / PANJIT): # 明确标出来,用户照着补一行规则就能识别,不用逐条猜。 for token in tokens: if looks_like_model(token) or re.search(r'\d', token): continue # 含数字的长串多半是型号,别拿它的碎片当品牌猜 for part in re.split(r'[//、,,;;||\s]+', token): candidate, _ = strip_brand_suffix(clean_value(part)) if (LATIN_ONLY.fullmatch(candidate) and len(candidate) >= 3 and not re.search(r'\d', candidate) and norm(candidate) not in PLACEHOLDER and not self.is_brand(candidate)): flags.append(f'疑似缺品牌:{candidate}') break else: continue break if not model and not no_model_marker: flags.append('型号未识别') def has_signal(values): return any(self.app_signal(value) for value in values) if not has_signal(app): # 缺项识别时位置仍可参考:前两位是品牌类型/出口享惠,应用场景与功能紧随其后。 excluded = {brand_raw, model_raw} for slot in (2, 3): if slot < len(tokens): token = tokens[slot] if token and token not in excluded and norm(token) not in PLACEHOLDER \ and not re.fullmatch(r'[\d.]+', token) and self.app_signal(token) \ and token not in app: app.append(token) if not has_signal(app): app.extend(t for t in tokens if any(k in t for k in ['用', '应用', '功能']) and t not in (brand_raw, model_raw) and t not in app) if not has_signal(app): app.extend(t for t in tokens if t and t not in (brand_raw, model_raw) and t not in app and self.hits_vocabulary(t)) a1, a2 = self.application(app) if a1 != a2: flags.append('应用二轮变更') if a2 in ('其他', '未识别'): flags.append('应用未识别') # 只有落在项目范围内的 HS 才有类目;范围外的记录留空,避免把无关 HS 误标成存储器。 c1, c2, c3, c4, description = '', '', '', '', '' if hs in self.cap: c1, c2 = '电容器', self.cap[hs] elif hs in MEMORY_HS: c1, c2 = '存储器', '其他存储器' if mapped and mapped[5]=='存储器': description = mapped[1] content = norm(' '.join(mapped[1:])) generation = re.search(r'(?:LP)?DDR([345])', content) if 'DDR' in content: c2, c3, c4 = 'DRAM', 'DDR', 'DDR' + generation[1] if generation else '其他DDR' elif 'SDRAM' in content: c2, c3 = 'DRAM', 'SDRAM' elif 'DRAM' in content or 'HBM' in content: c2, c3 = 'DRAM', '其他DRAM' elif 'FLASH' in content or '闪存' in content: c2, c3 = 'Flash', 'NAND' if 'NAND' in content else 'NOR' if 'NOR' in content else '其他Flash' if hs in MEMORY_HS and not mapped: flags.append('存储器分类未识别') country = unique(self.brand_country.get(norm(brand), ())) or '未识别' return dict(brand=brand, model=model, brand_country=country, c1=c1, c2=c2, c3=c3, c4=c4, description=description, app1=a1, app=a2, method=method, flags=';'.join(sorted(set(flags))), app_raw=' '.join(app), brand_raw=brand_raw, model_raw=model_raw, mapped=int(mapped is not None), brand_match_method=brand_match_method, brand_match_score=round(brand_match_score, 6), brand_match_key=brand_match_key, brand_candidates=brand_candidates) def application_order(labels) -> list[str]: """把任意标签集合按固定顺序排序,未列出的排最后。""" order = {label: index for index, label in enumerate(APPLICATION_LABELS)} return sorted(labels, key=lambda label: (order.get(label, 99), str(label))) APPLICATION_LABELS = ['先进计算', '汽车电子', '工业应用', '网络通信', '新能源', '高新电子', '消费电子', '通用领域', '其他']