"""Indexed fuzzy brand matching. Scores rank evidence, not probabilities.""" import re import unicodedata from collections import defaultdict from functools import lru_cache def compact(value): return re.sub(r'[^A-Z0-9\u3400-\u9fff]', '', unicodedata.normalize('NFKC', str(value or '')).upper()) def grams(text, size=2): return {text[i:i + size] for i in range(len(text) - size + 1)} def distance(a, b, limit): """Bounded Levenshtein distance, including insertions and deletions.""" if abs(len(a) - len(b)) > limit: return limit + 1 prev = list(range(len(b) + 1)) for i, ca in enumerate(a, 1): row = [i] for j, cb in enumerate(b, 1): row.append(min(row[-1] + 1, prev[j] + 1, prev[j - 1] + (ca != cb))) if min(row) > limit: return limit + 1 prev = row return prev[-1] class BrandMatcher: def __init__(self, aliases, excluded, allow_contains=True): self.allow_contains = allow_contains self.names = defaultdict(set) self.index = defaultdict(set) excluded = {compact(v) for v in excluded} for alias, labels in aliases.items(): key = compact(alias) if not key or key.isdigit() or key in excluded: continue self.names[key].update(labels) for key in self.names: for gram in grams(key): self.index[gram].add(key) @lru_cache(maxsize=32768) def match(self, variants, allow_typo=True): scores = {} for variant in variants: query = compact(variant) if not query or len(query) > 160: continue query_grams = grams(query) candidates = set() if query in self.names: candidates.add(query) for gram in query_grams: candidates.update(self.index.get(gram, ())) for key in candidates: score, method = 0, '' if key == query: score, method = 100.0, '标点归一' elif key in query: chinese = bool(re.search(r'[\u3400-\u9fff]', key)) minimum = 2 if chinese else 3 if len(key) < minimum: continue # Latin short names must be separate words, never inside model IDs. pattern = r'[^A-Z0-9\u3400-\u9fff]*'.join(map(re.escape, key)) original = unicodedata.normalize('NFKC', variant).upper() boundary = re.search(r'(?= .8: score, method = 100 * similarity, '包含匹配' elif boundary and (chinese or len(key) >= 4) and self.allow_contains: # 覆盖率不足 80% 时的兜底:整段里出现某个品牌名/别名即算命中, # 例如“日本兴亚”包含“兴亚”。拉丁名要求至少 4 个字符, # 避免 ST/AP 这类两三字母缩写在大段文本里误命中。 score, method = 100 * similarity, '子串包含' elif allow_typo and 4 <= len(query) <= 64 and len(key) >= 4: limit = int(max(len(query), len(key)) * .2 + 1e-9) if abs(len(key) - len(query)) > limit: continue # Any edit removes at most two shared bigrams. if len(query_grams & grams(key)) < max(1, len(query_grams) - 2 * limit): continue d = distance(query, key, limit) similarity = 1 - d / max(len(query), len(key)) if d <= limit and similarity >= .8: score, method = 100 * similarity, '拼写近似' if score: for label in self.names[key]: candidate = (score, key, method) current = scores.get(label) if current is None or candidate > current: scores[label] = candidate ranked = sorted(((label, *value) for label, value in scores.items()), key=lambda x: (-x[1], x[0])) if not ranked: return (), '', '', 0.0, () top = ranked[0] winners = tuple(row[0] for row in ranked if abs(row[1] - top[1]) < 1e-9) return winners, top[2], top[3], top[1], tuple(ranked[:5])