| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104 |
- """Indexed fuzzy brand matching. Scores rank evidence, not probabilities."""
- import re
- import unicodedata
- from collections import defaultdict
- from functools import lru_cache
- def compact(value):
- return re.sub(r'[^A-Z0-9\u3400-\u9fff]', '',
- unicodedata.normalize('NFKC', str(value or '')).upper())
- def grams(text, size=2):
- return {text[i:i + size] for i in range(len(text) - size + 1)}
- def distance(a, b, limit):
- """Bounded Levenshtein distance, including insertions and deletions."""
- if abs(len(a) - len(b)) > limit:
- return limit + 1
- prev = list(range(len(b) + 1))
- for i, ca in enumerate(a, 1):
- row = [i]
- for j, cb in enumerate(b, 1):
- row.append(min(row[-1] + 1, prev[j] + 1, prev[j - 1] + (ca != cb)))
- if min(row) > limit:
- return limit + 1
- prev = row
- return prev[-1]
- class BrandMatcher:
- def __init__(self, aliases, excluded, allow_contains=True):
- self.allow_contains = allow_contains
- self.names = defaultdict(set)
- self.index = defaultdict(set)
- excluded = {compact(v) for v in excluded}
- for alias, labels in aliases.items():
- key = compact(alias)
- if not key or key.isdigit() or key in excluded:
- continue
- self.names[key].update(labels)
- for key in self.names:
- for gram in grams(key):
- self.index[gram].add(key)
- @lru_cache(maxsize=32768)
- def match(self, variants, allow_typo=True):
- scores = {}
- for variant in variants:
- query = compact(variant)
- if not query or len(query) > 160:
- continue
- query_grams = grams(query)
- candidates = set()
- if query in self.names:
- candidates.add(query)
- for gram in query_grams:
- candidates.update(self.index.get(gram, ()))
- for key in candidates:
- score, method = 0, ''
- if key == query:
- score, method = 100.0, '标点归一'
- elif key in query:
- chinese = bool(re.search(r'[\u3400-\u9fff]', key))
- minimum = 2 if chinese else 3
- if len(key) < minimum:
- continue
- # Latin short names must be separate words, never inside model IDs.
- pattern = r'[^A-Z0-9\u3400-\u9fff]*'.join(map(re.escape, key))
- original = unicodedata.normalize('NFKC', variant).upper()
- boundary = re.search(r'(?<![A-Z0-9])' + pattern + r'(?![A-Z0-9])', original)
- similarity = len(key) / len(query)
- if boundary and similarity >= .8:
- score, method = 100 * similarity, '包含匹配'
- elif boundary and (chinese or len(key) >= 4) and self.allow_contains:
- # 覆盖率不足 80% 时的兜底:整段里出现某个品牌名/别名即算命中,
- # 例如“日本兴亚”包含“兴亚”。拉丁名要求至少 4 个字符,
- # 避免 ST/AP 这类两三字母缩写在大段文本里误命中。
- score, method = 100 * similarity, '子串包含'
- elif allow_typo and 4 <= len(query) <= 64 and len(key) >= 4:
- limit = int(max(len(query), len(key)) * .2 + 1e-9)
- if abs(len(key) - len(query)) > limit:
- continue
- # Any edit removes at most two shared bigrams.
- if len(query_grams & grams(key)) < max(1, len(query_grams) - 2 * limit):
- continue
- d = distance(query, key, limit)
- similarity = 1 - d / max(len(query), len(key))
- if d <= limit and similarity >= .8:
- score, method = 100 * similarity, '拼写近似'
- if score:
- for label in self.names[key]:
- candidate = (score, key, method)
- current = scores.get(label)
- if current is None or candidate > current:
- scores[label] = candidate
- ranked = sorted(((label, *value) for label, value in scores.items()),
- key=lambda x: (-x[1], x[0]))
- if not ranked:
- return (), '', '', 0.0, ()
- top = ranked[0]
- winners = tuple(row[0] for row in ranked if abs(row[1] - top[1]) < 1e-9)
- return winners, top[2], top[3], top[1], tuple(ranked[:5])
|