brandmatch.py 4.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104
  1. """Indexed fuzzy brand matching. Scores rank evidence, not probabilities."""
  2. import re
  3. import unicodedata
  4. from collections import defaultdict
  5. from functools import lru_cache
  6. def compact(value):
  7. return re.sub(r'[^A-Z0-9\u3400-\u9fff]', '',
  8. unicodedata.normalize('NFKC', str(value or '')).upper())
  9. def grams(text, size=2):
  10. return {text[i:i + size] for i in range(len(text) - size + 1)}
  11. def distance(a, b, limit):
  12. """Bounded Levenshtein distance, including insertions and deletions."""
  13. if abs(len(a) - len(b)) > limit:
  14. return limit + 1
  15. prev = list(range(len(b) + 1))
  16. for i, ca in enumerate(a, 1):
  17. row = [i]
  18. for j, cb in enumerate(b, 1):
  19. row.append(min(row[-1] + 1, prev[j] + 1, prev[j - 1] + (ca != cb)))
  20. if min(row) > limit:
  21. return limit + 1
  22. prev = row
  23. return prev[-1]
  24. class BrandMatcher:
  25. def __init__(self, aliases, excluded, allow_contains=True):
  26. self.allow_contains = allow_contains
  27. self.names = defaultdict(set)
  28. self.index = defaultdict(set)
  29. excluded = {compact(v) for v in excluded}
  30. for alias, labels in aliases.items():
  31. key = compact(alias)
  32. if not key or key.isdigit() or key in excluded:
  33. continue
  34. self.names[key].update(labels)
  35. for key in self.names:
  36. for gram in grams(key):
  37. self.index[gram].add(key)
  38. @lru_cache(maxsize=32768)
  39. def match(self, variants, allow_typo=True):
  40. scores = {}
  41. for variant in variants:
  42. query = compact(variant)
  43. if not query or len(query) > 160:
  44. continue
  45. query_grams = grams(query)
  46. candidates = set()
  47. if query in self.names:
  48. candidates.add(query)
  49. for gram in query_grams:
  50. candidates.update(self.index.get(gram, ()))
  51. for key in candidates:
  52. score, method = 0, ''
  53. if key == query:
  54. score, method = 100.0, '标点归一'
  55. elif key in query:
  56. chinese = bool(re.search(r'[\u3400-\u9fff]', key))
  57. minimum = 2 if chinese else 3
  58. if len(key) < minimum:
  59. continue
  60. # Latin short names must be separate words, never inside model IDs.
  61. pattern = r'[^A-Z0-9\u3400-\u9fff]*'.join(map(re.escape, key))
  62. original = unicodedata.normalize('NFKC', variant).upper()
  63. boundary = re.search(r'(?<![A-Z0-9])' + pattern + r'(?![A-Z0-9])', original)
  64. similarity = len(key) / len(query)
  65. if boundary and similarity >= .8:
  66. score, method = 100 * similarity, '包含匹配'
  67. elif boundary and (chinese or len(key) >= 4) and self.allow_contains:
  68. # 覆盖率不足 80% 时的兜底:整段里出现某个品牌名/别名即算命中,
  69. # 例如“日本兴亚”包含“兴亚”。拉丁名要求至少 4 个字符,
  70. # 避免 ST/AP 这类两三字母缩写在大段文本里误命中。
  71. score, method = 100 * similarity, '子串包含'
  72. elif allow_typo and 4 <= len(query) <= 64 and len(key) >= 4:
  73. limit = int(max(len(query), len(key)) * .2 + 1e-9)
  74. if abs(len(key) - len(query)) > limit:
  75. continue
  76. # Any edit removes at most two shared bigrams.
  77. if len(query_grams & grams(key)) < max(1, len(query_grams) - 2 * limit):
  78. continue
  79. d = distance(query, key, limit)
  80. similarity = 1 - d / max(len(query), len(key))
  81. if d <= limit and similarity >= .8:
  82. score, method = 100 * similarity, '拼写近似'
  83. if score:
  84. for label in self.names[key]:
  85. candidate = (score, key, method)
  86. current = scores.get(label)
  87. if current is None or candidate > current:
  88. scores[label] = candidate
  89. ranked = sorted(((label, *value) for label, value in scores.items()),
  90. key=lambda x: (-x[1], x[0]))
  91. if not ranked:
  92. return (), '', '', 0.0, ()
  93. top = ranked[0]
  94. winners = tuple(row[0] for row in ranked if abs(row[1] - top[1]) < 1e-9)
  95. return winners, top[2], top[3], top[1], tuple(ranked[:5])