| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849 |
- """Workbook rules, two-stage application matching and conservative G_MODEL parsing."""
- import re
- import unicodedata
- from collections import Counter, defaultdict
- from functools import lru_cache
- from openpyxl import load_workbook
- from .structure import (BRAND_QUALIFIER, CHINESE_STATUS, KEY_TOKEN, NO_BRAND, NO_MODEL,
- clean_value, label_kind, labels_in, looks_like_model,
- split_attributes, strip_brand_suffix, strip_model_suffix)
- from .brandmatch import BrandMatcher
- MEMORY_HS = {'8542321010', '8542321090', '8542329010', '8542329090'}
- # 这些词在品牌位上只表示“没有/不适用”,不能当成真实品牌参与自动扫描。
- PLACEHOLDER = {'无', '无品牌', '无牌', '不适用', '其他', '其它', 'NA', 'N/A', 'NONE', 'NIL',
- '待定', '未提供', '未知', '未识别', '-', '—', '/'}
- # 别名字符串里常见的说明性前缀,拆片段时要先去掉。
- ALIAS_LABEL = re.compile(
- r'(?:中文名称|外文名称|中文名|外文名|英文名称|品牌名称|商品名称|品牌|品名|品|BRAND|MODEL|NAME)'
- r'\s*[::=]\s*', re.I)
- ALIAS_SPLIT = re.compile(r'[,,、//;;||\s]+')
- # 拆出来的片段若只是通用词或国别,作为品牌线索只会制造误命中,不建索引。
- ALIAS_STOP = {
- '英文', '中文', '外文', '品牌', '品名', '名称', '商品名称', '型号', '等',
- '无中文名称', '无外文名称', '无中文品牌', '无外文品牌', '无中文名', '无外文名',
- '无品牌', '无牌', '其他', '其它', '未知', '不适用', '不详', '不祥',
- '株式会社', '有限公司', '公司', '集团', '电子', '电气', '半导体', '科技', '芯片',
- '中国', '日本', '韩国', '美国', '德国', '法国', '英国', '荷兰', '瑞士', '奥地利',
- '新加坡', '马来西亚', '泰国', '越南', '印度', '台湾', '中国台湾', '香港', '深圳',
- }
- def alias_tokens(raw):
- """把一个申报写法拆成可索引的片段:整串、去标签、括号内外、按分隔符切开的各项。
- 真实申报里经常写成“品:KOA等 中文名称:兴亚等”或“KOA/兴亚”“KOA,日本兴亚”,
- 只按整串建索引时,“品牌:日本兴亚”这类写法永远匹配不上;拆出片段后即可命中。
- """
- text = unicodedata.normalize('NFKC', str(raw or '')).strip()
- out = []
- def add(value):
- value = str(value or '').strip().strip('"\'“”‘’ \t,,;;/.')
- if value.endswith('等') and len(value) > 1:
- value = value[:-1]
- if value and value not in out:
- out.append(value)
- add(text)
- body = ALIAS_LABEL.sub(' ', text)
- add(body)
- for inner in re.findall(r'[((]([^))]*)[))]', body):
- add(inner)
- for part in ALIAS_SPLIT.split(re.sub(r'[((][^))]*[))]', ' ', body)):
- add(part)
- return out
- def norm(value):
- return re.sub(r'\s+', '', unicodedata.normalize('NFKC', str(value or ''))).upper()
- # 纯电气参数:容值、耐压、功率、尺寸、频率,这些不是型号。
- SPEC_LIKE = re.compile(
- r'^(?:\d+(?:\.\d+)?\s*(?:[nuµμpP]?F|[VvWwAa]|Hz|KHz|MHz|%|Ω|ohm)|\d{4}|\d+(?:\.\d+)?)$')
- # 像品牌名的写法(纯拉丁,不含中文),用来提示“映射表里可能缺这个品牌”。
- LATIN_ONLY = re.compile(r'[A-Za-z][A-Za-z0-9 .&+\-/]{1,30}')
- # 规则表里的兜底标签:不是真品牌,模糊/拼写/包含命中到它们一律不认。
- CATCH_ALL = {'Others', '其他', '其它', '其他品牌', '未识别', '未知', 'UNKNOWN'}
- # 明显是“品牌字段”的写法(带 牌/品牌 字样),用于在品牌认不出时仍能取到后面的型号。
- BRAND_FIELD = re.compile(r'(?:牌|品牌|商标)\s*$')
- def model_candidate(text, is_brand=None, app_signal=None):
- """判断一段文字能否当型号用:不是参数、不是别的标签的值、有字母也有数字。"""
- value = clean_value(text)
- if not value or SPEC_LIKE.match(value):
- return ''
- labels = list(labels_in(value))
- if labels and not any(kind in ('品牌', 'BRAND', '型号', 'MODEL') for kind, _ in labels):
- return ''
- if not (re.search(r'[A-Za-z]', value) and re.search(r'\d', value)):
- return ''
- if is_brand and is_brand(value):
- return ''
- if app_signal and app_signal(value):
- return ''
- return value
- def canon_label(value):
- """品牌标签的归并键:去掉括号说明与空白后大写。"""
- text = unicodedata.normalize('NFKC', str(value or '')).strip()
- text = re.sub(r'[((][^))]*[))]', '', text).strip()
- return re.sub(r'\s+', '', text).upper()
- def unique(values):
- values = set(values)
- return next(iter(values)) if len(values) == 1 else ''
- class Rules:
- def __init__(self, mapping, fields, country_system='ISO', fuzzy_contains=True):
- self.alias, self.brand_country, self.models = defaultdict(set), defaultdict(set), defaultdict(set)
- self.prefixes, self.first, self.second, self.cap, self.schema = [], {}, [], {}, {}
- self.conflicts = []
- self.fuzzy_contains = fuzzy_contains
- w = load_workbook(mapping, read_only=True, data_only=True)
- pairs = [(str(b).strip(), str(r).strip())
- for b, r in w['品牌名称映射规则'].iter_rows(min_row=2, values_only=True) if b and r]
- # 品牌标签存在 “MICRON/Micron”“AMD/AMD(XILINX)” 等同一品牌的多种写法,
- # 会让同一申报内容映射到多个标签而被整条判为冲突。先归并再建别名表。
- counts = Counter(label for label, _ in pairs)
- display = {}
- for label in counts:
- key = canon_label(label)
- current = display.get(key)
- if current is None or ((counts[label], -len(label), label)
- > (counts[current], -len(current), current)):
- display[key] = label
- self.label_map = {label: display[canon_label(label)] for label in counts}
- self.label_merges = sorted((a, b) for a, b in self.label_map.items() if a != b)
- full = defaultdict(set)
- for brand, raw in pairs:
- shown = self.label_map[brand]
- full[norm(raw)].add(shown)
- full[norm(brand)].add(shown)
- # 别名字符串常带说明前缀或并列写法。整串之外再按片段建索引,
- # “品牌:日本兴亚”这类只含品牌一部分的写法才能命中兴亚(KOA)。
- extra = defaultdict(set)
- for brand, raw in pairs:
- shown = self.label_map[brand]
- for token in alias_tokens(raw):
- key = norm(token)
- if not key or key in full or key in ALIAS_STOP:
- continue
- if len(key) < (2 if re.search(r'[\u3400-\u9fff]', key) else 3):
- continue
- extra[key].add(shown)
- for key, labels in extra.items():
- # 片段若指向多个品牌(说明是“电子”“英文”这类通用词),保留只会制造冲突。
- if len(labels) == 1:
- full[key] = labels
- self.alias = full
- # 记录“由片段推断出来”的键:型号表里的品牌中文名是显式品牌名,可以覆盖这类推断键。
- self.fragment_keys = {key for key, labels in extra.items() if len(labels) == 1}
- for brand, country in w['品牌国别映射规则'].iter_rows(min_row=2, values_only=True):
- if brand and country:
- label = str(brand).strip()
- self.brand_country[norm(self.label_map.get(label, label))].add(str(country).strip())
- for r in w['存储器型号分类映射规则'].iter_rows(min_row=2, values_only=True):
- en, cn, model, prefix, desc, c1, c2, c3, c4 = r
- if not model:
- continue
- brand = unique(self.alias[norm(en)]) or str(en or cn or '')
- for alias in (en, cn):
- if alias:
- # 型号表只补空位,或覆盖“由别名片段推断出来”的键(如 超微→SUPERMICRO);
- # 名称表里作为整串别名出现过的键不改写,避免同一写法被两个品牌认领。
- key = norm(alias)
- current = self.alias.get(key)
- if key in self.fragment_keys:
- self.alias[key] = {brand}
- elif not current or brand in current:
- self.alias[key].add(brand)
- record = (brand, str(desc or ''), str(c2 or ''), str(c3 or ''), str(c4 or ''),str(c1 or ''))
- self.models[norm(model)].add(record)
- if prefix:
- self.prefixes.append((norm(prefix), record))
- self.prefixes = sorted(set(self.prefixes), key=lambda x: -len(x[0]))
- self.prefix_lookup=defaultdict(set)
- for prefix,record in self.prefixes:self.prefix_lookup[prefix].add(record)
- # 四字片段索引:从任意一段文本里反查“里面藏着哪个已知型号”,
- # 用于“型号:K4B4G1646E等”“K4B4G1646E-BYK0”这类写法,避免全表逐条扫描。
- self.model_grams = defaultdict(set)
- for key in self.models:
- for i in range(len(key) - 3):
- self.model_grams[key[i:i + 4]].add(key)
- for r in w['应用领域映射规则(一轮匹配)'].iter_rows(min_row=2, values_only=True):
- if r[0] and r[1]:
- self.first.setdefault(norm(r[0]), set()).add(str(r[1]))
- for r in w['应用领域与功能映射规则(二轮匹配)'].iter_rows(min_row=2, values_only=True):
- if r[0] not in ('其他', '通用领域') and r[1]:
- self.second.append((r[0], [norm(k) for k in str(r[1]).split('、')]))
- for r in w['电容器映射规则'].iter_rows(min_row=2, values_only=True):
- if r[2]:
- self.cap[str(r[2])[:10]] = str(r[1])
- w.close()
- for kind, mapping in [('品牌', self.alias), ('品牌国别', self.brand_country), ('型号', self.models), ('应用一轮', self.first)]:
- for key, values in mapping.items():
- if len(values) > 1:
- self.conflicts.append((kind, key, repr(sorted(values))))
- for original, merged in self.label_merges:
- self.conflicts.append(('品牌标签归并', original, f'归并为 {merged}'))
- self.brand_matcher = BrandMatcher(self.alias, PLACEHOLDER | {
- '无中文品牌', '无外文品牌', '无中文名称', '无外文名称'}, allow_contains=fuzzy_contains)
- self._build_reverse_index()
- w = load_workbook(fields, read_only=True, data_only=True)
- for r in w['字段规则_申报要素'].iter_rows(min_row=3, values_only=True):
- if r[0]:
- self.schema[str(r[0])] = [str(x or '') for x in r[2:]]
- country_candidates=defaultdict(set)
- self.countries, self.ports = {}, {}
- for r in w['映射规则_国别地区代码'].iter_rows(min_row=2, values_only=True):
- if not r[6]:
- continue
- keys = [r[0]] if country_system == 'GEC' else [r[1], r[2], r[3]]
- for key in keys + [r[6]]:
- if key is not None:
- if str(key) in ['不适用','无']:continue
- country_candidates[norm(key)].add(str(r[6]))
- if str(key).isdigit():
- country_candidates[str(key).zfill(3)].add(str(r[6]))
- for key,values in country_candidates.items():
- if len(values)==1:self.countries[key]=next(iter(values))
- else:
- self.countries[key]='映射冲突:'+key
- self.conflicts.append(('国家地区',key,repr(sorted(values))))
- for code, cn, en, country in w['映射规则_港口代码'].iter_rows(min_row=2, values_only=True):
- if code:
- self.ports[norm(code)] = (str(cn or en or code), str(country or '未识别'))
- w.close()
- def country(self, raw):
- return self.countries.get(norm(raw), f'未映射:{raw}' if raw else '未识别')
- def port(self, raw):
- key = norm(raw)
- return self.ports.get(key, (f'未映射:{raw}', '未识别')) if key else ('未识别', '未识别')
- # ---------- 品牌 / 型号取词 ----------
- @staticmethod
- def _clean(raw):
- text = unicodedata.normalize('NFKC', str(raw or '')).strip()
- return text.strip('"\'“”‘’ \t').strip()
- def brand_variants(self, raw):
- """按可靠性从高到低给出品牌候选写法:整体、去“牌”、括号内外、枚举各项。"""
- text = self._clean(raw)
- text = re.sub(r'^(?:品牌|BRAND)\s*[::=]\s*', '', text, flags=re.I)
- out = []
- def add(value):
- value = str(value or '').strip().strip('"\'“”‘’ \t').strip()
- if value and value not in out:
- out.append(value)
- add(text)
- if text.endswith('牌') and len(text) > 1:
- add(text[:-1])
- match = re.match(r'^(.*?)[((]([^))]*)[))](.*)$', text)
- if match:
- add(match.group(1) + match.group(3))
- add(match.group(1))
- add(match.group(2))
- if match.group(2).endswith('牌') and len(match.group(2)) > 1:
- add(match.group(2)[:-1])
- flat = re.sub(r'[((][^))]*[))]', ' ', text)
- for part in re.split(r'[,,、//;;||]+', flat):
- add(part)
- for part in re.split(r'[\s_]+', text):
- add(part)
- return out
- def _build_reverse_index(self):
- """为"规则表里的长申报内容包含 G_MODEL 写法"建索引。
- 品牌表里有很多长串,比如 `PANJIT牌/强茂牌`、`村田制作所(MURATA)`、
- `NEXPERIA、DIODES、PANJIT/无中文名称`。反过来查"这段文字出现在哪条规则里",
- 用 3 字片倒排 + 逐条验证,避免每次全表扫描。
- """
- self._rev_keys = [] # [(规范化的键, 该键对应的品牌集合)]
- self._rev_grams = defaultdict(set) # 二字片 -> 键序号集合(中文两字就是完整品牌)
- for key, labels in self.alias.items():
- packed = norm(key)
- if len(packed) < 2:
- continue
- index = len(self._rev_keys)
- self._rev_keys.append((packed, frozenset(labels)))
- for gram in {packed[i:i + 2] for i in range(len(packed) - 1)}:
- self._rev_grams[gram].add(index)
- @lru_cache(maxsize=16384)
- def reverse_brand_labels(self, text):
- """返回"规则表里的哪条申报内容包含这段文字"所对应的品牌。
- 例:'PANJIT' 会被 `PANJIT牌/强茂牌` 命中(长申报内容里含英文名)。
- 只认"除这段文字外没有别的已知品牌"的那几条规则——像
- `NEXPERIA、DIODES、PANJIT/无中文名称` 这种多品牌混写不算,
- 否则会把 PANJIT 认成 Nexperia。
- """
- packed = norm(text)
- # 中文两字就可能是完整品牌(村田/强茂),拉丁字母要四个以上才不至于乱撞(ST/AP 不算)
- minimum = 2 if re.search(r'[\u4e00-\u9fff]', packed) else 4
- if len(packed) < minimum:
- return frozenset()
- candidates = None
- for gram in {packed[i:i + 2] for i in range(len(packed) - 1)}:
- bucket = self._rev_grams.get(gram)
- if not bucket:
- return frozenset()
- candidates = set(bucket) if candidates is None else candidates & bucket
- if not candidates:
- return frozenset()
- if not candidates:
- return frozenset()
- labels = set()
- for index in candidates:
- packed_key, key_labels = self._rev_keys[index]
- if packed not in packed_key:
- continue
- if key_labels <= CATCH_ALL:
- continue # 兜底标签不算证据
- # 只认"这条规则差不多就是这个品牌名"的行:留 8 个字符的余量给 牌/无中文名称 这类后缀。
- # 像 COMCHIPONPANJITPYSEMTECH…(多个品牌连写)这种长串一律不认。
- if len(packed_key) > len(packed) + 8:
- continue
- if not self._single_brand_rule(packed_key, packed):
- continue
- labels |= key_labels
- return frozenset(labels)
- def _single_brand_rule(self, packed_key, packed):
- """这条规则除了这段文字,还有没有写着别的已知品牌。"""
- rest = packed_key.replace(packed, '', 1)
- for piece in re.split(r'[、,,//;;||()()\s::]+', rest):
- piece = piece.strip()
- if len(piece) < 2 or re.fullmatch(r'[\d.\-]+', piece):
- continue
- found = self.alias.get(piece) or self.alias.get(piece.rstrip('牌'))
- if found:
- return False
- return True
- def brand_detail(self, raw, allow_typo=True):
- """Return labels, matched key, warning, method, score, ranked candidates."""
- if norm(raw) in PLACEHOLDER or BRAND_QUALIFIER.fullmatch(str(raw or '')):
- return set(), '', '', '', 0.0, ()
- variants = self.brand_variants(raw)
- whole = self.alias.get(norm(variants[0])) if variants else None
- if whole:
- return (set(whole) if len(whole) == 1 else set(), variants[0],
- '' if len(whole) == 1 else '品牌映射冲突', '整体精确', 100.0,
- tuple((v, 100.0, variants[0], '整体精确') for v in sorted(whole)))
- hits = []
- hit = ''
- for variant in variants:
- if norm(variant) in PLACEHOLDER or BRAND_QUALIFIER.fullmatch(variant):
- continue
- found = self.alias.get(norm(variant))
- if found:
- hits.append(set(found))
- if not hit:
- hit = variant
- if hits:
- agreed = set(hits[0]).intersection(*hits[1:])
- candidates = tuple((v, 100.0, hit, '取词精确') for v in sorted(set.union(*hits)))
- return (agreed if len(agreed) == 1 else set(), hit,
- '' if len(agreed) == 1 else '品牌映射冲突', '取词精确', 100.0, candidates)
- labels, hit, method, score, candidates = self.brand_matcher.match(tuple(variants), allow_typo)
- catch_all_hit = bool(labels) and set(labels) <= CATCH_ALL
- if labels and catch_all_hit and method in ('拼写近似', '包含匹配', '子串包含', '标点归一'):
- # 拼错一个字母就落到 Others/其他 这种兜底标签上,几乎一定是认错了
- # (实测 PANJIT 被拼写近似到表里的错行 PANJJIT → Others)。宁可继续往下找。
- labels = set()
- if not labels:
- # 反向包含:规则表里的申报内容是一长串(可能中英混写),只要这段文字
- # 整段出现在里面就算命中。例:'PANJIT' ⊂ 'PANJIT牌/强茂牌'。
- reversed_labels = None
- for variant in variants:
- text = norm(variant)
- # 太短的拉丁串(ST、AP)先不看;中文两字可能是完整品牌
- minimum = 2 if re.search(r'[\u4e00-\u9fff]', text) else 4
- if len(text) < minimum or re.search(r'\d', text):
- continue
- found = self.reverse_brand_labels(text)
- if not found:
- continue
- reversed_labels = set(found) if reversed_labels is None else reversed_labels & set(found)
- if reversed_labels and len(reversed_labels) == 1:
- return (reversed_labels, hit, '', '反向包含', 90.0,
- tuple((v, 90.0, hit, '反向包含') for v in sorted(reversed_labels)))
- if reversed_labels and len(reversed_labels) > 1:
- return set(), hit, '品牌反向包含冲突', '反向包含', 90.0, ()
- if catch_all_hit:
- return set(), hit, '品牌近似命中兜底标签,已忽略', method, score, candidates
- return (set(labels) if len(labels) == 1 else set(), hit,
- '品牌映射冲突' if len(labels) > 1 else '', method, score, candidates)
- def resolve_brand(self, raw):
- return self.brand_detail(raw)[:3]
- def is_brand(self, raw):
- # Avoid spelling guesses on unlabelled model IDs containing digits.
- labels, _, note, *_ = self.brand_detail(raw, allow_typo=not bool(re.search(r'\d', str(raw))))
- return bool(labels and not note)
- def model_inside(self, tokens, brand):
- """在整段文本里找回已知型号:文本包含某个已知型号即算命中,最长优先。
- 典型场景是“型号:K4B4G1646E等”“K4B4G1646E-BYK0”“K4B4G1646E K4A8G165WB”,
- 原逻辑只做整串/去“等”/空格拆分/前缀,这类型号会整条判为未识别。
- """
- best = None
- for token in tokens:
- text = norm(token)
- if len(text) < 4:
- continue
- candidates = set()
- for i in range(len(text) - 3):
- candidates |= self.model_grams.get(text[i:i + 4], set())
- for key in candidates:
- if key not in text:
- continue
- records = {r for r in self.models[key] if not brand or norm(r[0]) == norm(brand)}
- if records and (best is None or len(key) > len(best[0])):
- best = (key, records)
- return best
- def model_variants(self, raw):
- """型号候选写法:整体、括号内外、按空白/下划线、按逗号顿号枚举。"""
- text = self._clean(raw)
- text = re.sub(r'^(?:型号|MODEL)\s*[::=]\s*', '', text, flags=re.I)
- out = []
- def add(value):
- value = str(value or '').strip().strip('"\'“”‘’ \t').strip()
- if value and value not in out:
- out.append(value)
- add(text)
- match = re.match(r'^(.*?)[((]([^))]*)[))](.*)$', text)
- if match:
- add(match.group(1))
- add(match.group(2))
- add(match.group(1) + match.group(3))
- for part in re.split(r'[\s_]+', text):
- add(part)
- for part in re.split(r'[,,、]+', text):
- add(part)
- return out
- def app_signal(self, token):
- """该字段文本能否支撑一个具体应用领域(“其他/未识别”不算信号)。"""
- text = norm(str(token or '').replace('_', ' '))
- if not text or text in PLACEHOLDER:
- return False
- label = unique(self.first.get(text, ()))
- if label and label not in ('其他', '未识别', '通用领域'):
- return True
- return any(key in text for _, keys in self.second for key in keys)
- def hits_vocabulary(self, token):
- """该字段文本是否落在应用领域的已知词表里。"""
- text = norm(str(token or '').replace('_', ' '))
- if not text or text in PLACEHOLDER:
- return False
- if text in self.first:
- return True
- return any(key in text for _, keys in self.second for key in keys)
- def application(self, texts):
- text = norm(' '.join(texts).replace('_', ' '))
- if not text:
- return '未识别', '未识别'
- first = unique(self.first.get(text, ()))
- if not first:
- hits = set()
- for part in texts:
- hits.update(self.first.get(norm(part.replace('_', ' ')), ()))
- first = unique(hits) or ('通用领域' if hits else '其他')
- hits = set()
- for category, keys in self.second:
- candidate = text
- # Exclusions are phrase-level: satellite communication is high-tech;
- # computer systems are advanced computing, ordinary computers consumer.
- if category == '网络通信':
- candidate = candidate.replace('卫星通信', '')
- if category == '消费电子':
- candidate = candidate.replace('计算机系统', '')
- if category == '高新电子':
- candidate = candidate.replace('无人飞机', '')
- if any((bool(re.search(r'(?<![A-Z])AI(?![A-Z])', candidate)) if k == 'AI' else k in candidate) for k in keys):
- hits.add(category)
- if '通用' in text:
- hits.add('通用领域')
- return first, ('通用领域' if len(hits) > 1 else unique(hits) or '其他')
- @lru_cache(maxsize=8192)
- def parse(self, hs, raw):
- flags, brand_raw, model_raw, app = [], '', '', []
- label_brand_fallback = ''
- tokens, split_mode, split_warning = split_attributes(
- raw, is_brand=self.is_brand)
- if split_mode in ('space_inferred', 'underscore_inferred'):
- labels, _, note, kind, score, _ = self.brand_detail(raw, allow_typo=False)
- if labels and not note and kind == '标点归一' and score == 100:
- tokens, split_mode, split_warning = [str(raw)], 'primary', ''
- if split_warning:
- flags.append(split_warning)
- if re.search(r'(?:^|[||;;])\s*[//]\s*(?:$|[||;;])', str(raw or '')):
- flags.append('斜杠占位字段')
- schema = self.schema.get(hs, [])
- expected = len([x for x in schema if x])
- brand_slot = next((i for i, label in enumerate(schema)
- if label.startswith('品牌') and '类型' not in label), None)
- model_slot = next((i for i, label in enumerate(schema) if label == '型号'), None)
- first_is_status = bool(tokens and re.fullmatch('[0-4]', tokens[0]))
- chinese_status = bool(tokens and CHINESE_STATUS.fullmatch(tokens[0]) and
- brand_slot is not None and brand_slot < len(tokens) and
- self.is_brand(tokens[brand_slot]))
- extra_only_empty = len(tokens) > expected and not any(tokens[expected:])
- short_but_anchored = (len(tokens) < expected and brand_slot is not None and
- model_slot is not None and model_slot < len(tokens) and
- self.is_brand(tokens[brand_slot]))
- # Accept missing optional tail only when the brand position is independently anchored.
- aligned = len(tokens) == expected or extra_only_empty or short_but_anchored
- complete = bool(schema and aligned and
- (first_is_status or chinese_status))
- if complete:
- if extra_only_empty:
- flags.append('模板末尾空段')
- if short_but_anchored:
- flags.append('模板末尾字段缺失待核查')
- for label, token in zip(schema, tokens):
- if label.startswith('品牌') and '类型' not in label:
- brand_raw = token
- if label == '型号':
- model_raw = token
- if any(k in label for k in ['用途', '应用', '功能']):
- app.append(token)
- method = '顺序提取'
- # 只有"其它位置确实有真实品牌"才否定位置提取;“无/不适用”这类占位词不算线索,
- # 否则品牌位留空、GTIN/CAS 写“无”的记录会把已经取到的型号也一起丢掉。
- other_brand = any(self.is_brand(t) and norm(t) not in PLACEHOLDER for t in tokens)
- if brand_raw and not self.is_brand(brand_raw) and other_brand:
- brand_raw,model_raw,app='','',[]
- method='缺项识别'
- flags.append('顺序位置与品牌线索冲突,转缺项识别')
- else:
- method = '缺项识别'
- # Labels take precedence; numbered prefixes such as "1、型号:" are accepted.
- label_brands, label_models = [], []
- for index, token in enumerate(tokens):
- for label, value in labels_in(token):
- if label in ('品牌', 'BRAND'):
- label_brands.append(value)
- elif label in ('型号', 'MODEL'):
- label_models.append(value)
- else: app.append(value)
- # 键和值各占一段:品牌|MURATA|型号|GRM188…
- if KEY_TOKEN.fullmatch(token.strip()):
- nxt = tokens[index + 1] if index + 1 < len(tokens) else ''
- value = clean_value(nxt) if nxt and not KEY_TOKEN.fullmatch(nxt.strip()) else ''
- kind = label_kind(token.strip())
- if value and kind == '品牌':
- label_brands.append(value)
- elif value and kind == '型号':
- label_models.append(value)
- brand_label_conflict = len({norm(value) for value in label_brands}) > 1
- model_label_conflict = len({norm(value) for value in label_models}) > 1
- if brand_label_conflict:
- brand_raw = ''
- flags.append('品牌标签冲突')
- elif label_brands:
- brand_raw = label_brands[0]
- # 标签里给了品牌:能匹配上就锁死,后面的扫描不再覆盖。
- if NO_BRAND.fullmatch(brand_raw) or BRAND_QUALIFIER.fullmatch(brand_raw):
- flags.append('无品牌标记')
- else:
- probe = self.brand_detail(brand_raw)
- if probe[0] and not probe[2]:
- flags.append('品牌标签锁定')
- elif probe[2]:
- # 一句话里写了两个不同品牌:保留原文,交给后面统一判冲突,不擅自取一个。
- pass
- else:
- head = re.split(r'[,,;;。]', brand_raw)[0].strip()
- if head and head != brand_raw:
- head_probe = self.brand_detail(head)
- if head_probe[0] and not head_probe[2]:
- # “品牌:TDK,用于手机”这种句子值:逗号后面是说明,取前半段
- brand_raw = head
- flags.append('品牌标签锁定')
- probe = head_probe
- else:
- probe = ()
- if not probe:
- flags.append('品牌标签未命中')
- label_brand_fallback, brand_raw = brand_raw, ''
- if model_label_conflict:
- # 一条记录里同时写了“型号:”“规格型号:”等,按“多型号取首个”处理,不整条丢弃。
- model_raw = label_models[0]
- flags.append('多型号取首个')
- elif label_models:
- model_raw = label_models[0]
- # 明确写“无型号/无规格”的,视为已经说清楚,不再去别处凑一个型号,也不算识别失败。
- no_model_marker = bool(model_raw and NO_MODEL.fullmatch(model_raw))
- if no_model_marker:
- model_raw = ''
- flags.append('无型号标记')
- if not brand_raw and not brand_label_conflict:
- brand_raw = next((t for t in tokens if NO_BRAND.fullmatch(t)), '')
- if not brand_raw and not brand_label_conflict:
- # 逐字段按“整体 -> 去牌 -> 括号内外 -> 枚举项”解析,排除“无/不适用”等占位词。
- found = []
- for index, token in enumerate(tokens):
- if not token or norm(token) in PLACEHOLDER or BRAND_QUALIFIER.fullmatch(token) \
- or re.fullmatch(r'[\d.]+', token):
- continue
- # Explicit model/application fields cannot supply a brand by fuzzy scanning.
- token_labels = list(labels_in(token))
- if token_labels and not any(label in ('品牌', 'BRAND') for label, _ in token_labels):
- continue
- allow_typo = (index == brand_slot or
- (not re.search(r'\d', token) and not self.hits_vocabulary(token)))
- labels, hit, note, kind, score, candidates = self.brand_detail(token, allow_typo=allow_typo)
- if labels and not note:
- found.append((score, index, token, labels))
- if found:
- best = max(row[0] for row in found)
- top = [row for row in found if row[0] == best]
- distinct = set.union(*(row[3] for row in top))
- if len(distinct) == 1:
- brand_raw = top[0][2]
- else:
- flags.append('品牌位置不确定')
- if not brand_raw and label_brand_fallback:
- # 标签里写过、但匹配不上,且别处也没找到:保留原文,便于核查到底是哪种写法没覆盖。
- brand_raw = label_brand_fallback
- # “ABACUS等牌”里的“等牌”不是名字的一部分,去掉再查表,否则同品牌会被劈成两条。
- brand_raw, brand_had_deng = strip_brand_suffix(brand_raw)
- if brand_had_deng:
- flags.append('品牌带“等”后缀')
- brand_match_method, brand_match_score, brand_match_key, brand_candidates = '', 0.0, '', ()
- brand_set = set()
- if not NO_BRAND.fullmatch(brand_raw):
- brand_set, brand_hit, brand_note, brand_match_method, brand_match_score, brand_candidates = self.brand_detail(brand_raw)
- brand_match_key = brand_hit
- if brand_note:
- flags.append(brand_note)
- elif brand_match_method in ('标点归一', '包含匹配', '拼写近似'):
- flags.append('品牌模糊匹配')
- elif brand_match_method == '子串包含':
- flags.append('品牌包含匹配')
- elif brand_hit and brand_hit != brand_raw:
- flags.append('品牌括号或枚举取词')
- brand = unique(brand_set)
- if NO_BRAND.fullmatch(brand_raw) or BRAND_QUALIFIER.fullmatch(brand_raw):
- flags.append('无品牌标记')
- if not model_raw and not model_label_conflict and not no_model_marker:
- found = [(i, t) for i, t in enumerate(tokens) if norm(t) in self.models]
- if found: model_raw = found[0][1]
- if len(found) > 1: flags.append('多型号取首个')
- if not model_raw and brand_raw and not model_label_conflict and not no_model_marker:
- for i, token in enumerate(tokens):
- # 带用途/应用等标签的段不能供型号:否则“用途:5G基站”会被当成型号。
- token_labels = list(labels_in(token))
- if token_labels and not any(label in ('品牌', 'BRAND', '型号', 'MODEL')
- for label, _ in token_labels):
- continue
- variants = self.brand_variants(token)
- hit = next((v for v in variants if norm(v) == norm(brand_raw)), '')
- if brand_match_key and brand_match_key in token and self.resolve_brand(token)[0] == {brand}:
- hit = brand_match_key
- if not hit and brand and self.resolve_brand(token)[0] == {brand}:
- hit = brand_match_key if brand_match_key in token else token
- if not hit:
- continue
- tail = clean_value(token[token.find(hit) + len(hit):]) if hit in token else ''
- following = tokens[i + 1] if i + 1 < len(tokens) else ''
- for candidate in (tail, following):
- value = model_candidate(candidate, self.is_brand, self.app_signal)
- if value:
- model_raw = value
- flags.append('品牌后型号推断')
- break
- break
- if not model_raw and not model_label_conflict and not no_model_marker and brand:
- # 型号写在品牌前面("GRM188R71H104KA93D MURATA"):只认同品牌紧挨着的那一段,
- # 且必须明显像料号,避免把 100nF、0402、5G基站 这类参数或用途当成型号。
- anchor = next((i for i, t in enumerate(tokens)
- if norm(t) == norm(brand_raw)), None)
- if anchor is None and brand_match_key:
- anchor = next((i for i, t in enumerate(tokens)
- if brand_match_key in t), None)
- if anchor:
- cand = clean_value(tokens[anchor - 1]) if anchor > 0 else ''
- if (looks_like_model(cand) and cand != brand_raw
- and not self.is_brand(cand) and not self.app_signal(cand)
- and norm(cand) not in PLACEHOLDER):
- model_raw = cand
- flags.append('品牌前型号推断')
- if not model_raw and not model_label_conflict and not no_model_marker and brand:
- # 型号和品牌挤在同一段里("GRM188… MURATA"):把每段再拆成片段,挑明显像料号的那一个。
- for token in tokens:
- for part in re.split(r'[\s_\u3000、,,()()]+', token):
- cand = clean_value(part)
- if (looks_like_model(cand) and not self.is_brand(cand)
- and not self.app_signal(cand) and norm(cand) not in PLACEHOLDER
- and norm(cand) != norm(brand_raw)):
- model_raw = cand
- flags.append('型号片段提取')
- break
- if model_raw:
- break
- # “A5E30295495R2W等型”里的“等型”不是型号的一部分:去掉,并按“多型号取首个”记账。
- if not model_raw and not no_model_marker and not model_label_conflict:
- # 品牌认不出来也不能把型号一起丢掉:如果前一段明显是品牌字段(带“牌/品牌”),
- # 紧跟其后的那一段又像料号,就照取(型号本身不依赖品牌表)。
- for index, token in enumerate(tokens[:-1]):
- if BRAND_FIELD.search(clean_value(token) or token.strip()):
- nxt = tokens[index + 1]
- value = clean_value(nxt)
- if (looks_like_model(value) and value != brand_raw
- and not self.is_brand(value) and not self.app_signal(value)):
- model_raw = value
- flags.append('品牌字段后型号推断')
- break
- # “A5E30295495R2W等型”里的“等型”不是型号的一部分:去掉,并按“多型号取首个”记账。
- model_raw, model_had_deng = strip_model_suffix(model_raw)
- if model_had_deng:
- flags.append('多型号取首个')
- model = norm(model_raw)
- if model and model not in self.models:
- if model.endswith('等'):
- model = model[:-1]
- flags.append('多型号取首个')
- if any(mark in model_raw for mark in ',,、'):
- first = norm(re.split(r'[,,、]+', model_raw, maxsplit=1)[0])
- if first:
- model = first
- flags.append('多型号取首个')
- elif re.search(r'[\s_]', model_raw) and model not in self.models:
- words = [norm(part) for part in re.split(r'[\s_]+', model_raw) if part]
- matches_in_words = [part for part in words if part in self.models]
- if len(matches_in_words) == 1:
- model = matches_in_words[0]
- flags.append('空格分隔型号推断')
- if not model and hs in MEMORY_HS and not model_label_conflict and not no_model_marker:
- # 型号可能和品牌挤在同一段(如“长江存储(YMTC) YMTC-XX”),逐字段找回。
- for token in tokens:
- for variant in self.model_variants(token):
- key = norm(variant)
- if not key or key in PLACEHOLDER or key not in self.models:
- continue
- hits = self.models[key]
- if brand:
- hits = {r for r in hits if norm(r[0]) == norm(brand)}
- if len(hits) == 1:
- model_raw, model = variant, key
- flags.append('型号候选扫描')
- break
- if model:
- break
- matches = self.models.get(model, set())
- if brand: matches = {r for r in matches if norm(r[0]) == norm(brand)}
- if not matches and model and hs in MEMORY_HS:
- for length in range(len(model),0,-1):
- hits={r for r in self.prefix_lookup.get(model[:length],()) if not brand or norm(r[0])==norm(brand)}
- if hits:
- matches=hits;flags.append('型号前缀匹配');break
- # 最后兜底:文本里含某个已知型号就算命中(“xxx等”“xxx-yyy”)。
- # 只在型号表覆盖的 HS 上做,避免把无关商品的编码当型号。
- if not matches and hs in MEMORY_HS and self.fuzzy_contains:
- hit = self.model_inside(tokens, brand)
- if hit:
- model, matches = hit
- flags.append('型号包含匹配')
- # Different descriptions that imply the same taxonomy are still a mapping conflict.
- mapped = next(iter(matches)) if len(matches) == 1 else None
- if len(matches) > 1: flags.append('型号映射冲突')
- if mapped and not brand and not any('品牌' in f and ('冲突' in f or '不确定' in f) for f in flags):
- brand = mapped[0]
- brand_match_method, brand_match_score = '型号反推', 0.0
- if not brand and not NO_BRAND.fullmatch(brand_raw) and not BRAND_QUALIFIER.fullmatch(brand_raw):
- flags.append('品牌未识别')
- # 字段里有像品牌名的拉丁写法、但映射表查不到(如 ABACUS / PANJIT):
- # 明确标出来,用户照着补一行规则就能识别,不用逐条猜。
- for token in tokens:
- if looks_like_model(token) or re.search(r'\d', token):
- continue # 含数字的长串多半是型号,别拿它的碎片当品牌猜
- for part in re.split(r'[//、,,;;||\s]+', token):
- candidate, _ = strip_brand_suffix(clean_value(part))
- if (LATIN_ONLY.fullmatch(candidate) and len(candidate) >= 3
- and not re.search(r'\d', candidate)
- and norm(candidate) not in PLACEHOLDER
- and not self.is_brand(candidate)):
- flags.append(f'疑似缺品牌:{candidate}')
- break
- else:
- continue
- break
- if not model and not no_model_marker: flags.append('型号未识别')
- def has_signal(values):
- return any(self.app_signal(value) for value in values)
- if not has_signal(app):
- # 缺项识别时位置仍可参考:前两位是品牌类型/出口享惠,应用场景与功能紧随其后。
- excluded = {brand_raw, model_raw}
- for slot in (2, 3):
- if slot < len(tokens):
- token = tokens[slot]
- if token and token not in excluded and norm(token) not in PLACEHOLDER \
- and not re.fullmatch(r'[\d.]+', token) and self.app_signal(token) \
- and token not in app:
- app.append(token)
- if not has_signal(app):
- app.extend(t for t in tokens if any(k in t for k in ['用', '应用', '功能'])
- and t not in (brand_raw, model_raw) and t not in app)
- if not has_signal(app):
- app.extend(t for t in tokens if t and t not in (brand_raw, model_raw)
- and t not in app and self.hits_vocabulary(t))
- a1, a2 = self.application(app)
- if a1 != a2: flags.append('应用二轮变更')
- if a2 in ('其他', '未识别'): flags.append('应用未识别')
- # 只有落在项目范围内的 HS 才有类目;范围外的记录留空,避免把无关 HS 误标成存储器。
- c1, c2, c3, c4, description = '', '', '', '', ''
- if hs in self.cap:
- c1, c2 = '电容器', self.cap[hs]
- elif hs in MEMORY_HS:
- c1, c2 = '存储器', '其他存储器'
- if mapped and mapped[5]=='存储器':
- description = mapped[1]
- content = norm(' '.join(mapped[1:]))
- generation = re.search(r'(?:LP)?DDR([345])', content)
- if 'DDR' in content:
- c2, c3, c4 = 'DRAM', 'DDR', 'DDR' + generation[1] if generation else '其他DDR'
- elif 'SDRAM' in content: c2, c3 = 'DRAM', 'SDRAM'
- elif 'DRAM' in content or 'HBM' in content: c2, c3 = 'DRAM', '其他DRAM'
- elif 'FLASH' in content or '闪存' in content:
- c2, c3 = 'Flash', 'NAND' if 'NAND' in content else 'NOR' if 'NOR' in content else '其他Flash'
- if hs in MEMORY_HS and not mapped: flags.append('存储器分类未识别')
- country = unique(self.brand_country.get(norm(brand), ())) or '未识别'
- return dict(brand=brand, model=model, brand_country=country, c1=c1, c2=c2, c3=c3, c4=c4,
- description=description, app1=a1, app=a2, method=method, flags=';'.join(sorted(set(flags))),
- app_raw=' '.join(app), brand_raw=brand_raw, model_raw=model_raw, mapped=int(mapped is not None),
- brand_match_method=brand_match_method, brand_match_score=round(brand_match_score, 6),
- brand_match_key=brand_match_key, brand_candidates=brand_candidates)
- def application_order(labels) -> list[str]:
- """把任意标签集合按固定顺序排序,未列出的排最后。"""
- order = {label: index for index, label in enumerate(APPLICATION_LABELS)}
- return sorted(labels, key=lambda label: (order.get(label, 99), str(label)))
- APPLICATION_LABELS = ['先进计算', '汽车电子', '工业应用', '网络通信', '新能源', '高新电子', '消费电子', '通用领域', '其他']
|