feat：1.敏感词检测忽略html标签；2.添加敏感词功能

src/main/java/cn/seecoder/fdroidrepository/DataObject/Enum/ResultCode.java

@@ -12,7 +12,9 @@ public enum ResultCode {
     // Bug 相关的错误代码
 
     // Post 相关的错误代码
-    POST_NON_EXIST(40001, "对应的POST不存在或被删除");
+    POST_NON_EXIST(40001, "对应的POST不存在或被删除"),
+    // 敏感词相关错误代码
+    SENSITIVE_ADD_ERROR(50001, "添加敏感词错误");
     private Integer code;
     private String message;
     ResultCode(Integer code, String message) {

src/main/java/cn/seecoder/fdroidrepository/Service/SensitiveWordService.java

@@ -0,0 +1,8 @@
+package cn.seecoder.fdroidrepository.Service;
+
+import cn.seecoder.fdroidrepository.DataObject.VO.Response;
+
+public interface SensitiveWordService {
+    Response list();
+    Response add(String word);
+}

src/main/java/cn/seecoder/fdroidrepository/Service/ServiceImpl/SensitiveWordServiceImpl.java

@@ -0,0 +1,42 @@
+package cn.seecoder.fdroidrepository.Service.ServiceImpl;
+
+import cn.seecoder.fdroidrepository.DataObject.Enum.ResultCode;
+import cn.seecoder.fdroidrepository.DataObject.VO.Response;
+import cn.seecoder.fdroidrepository.Service.SensitiveWordService;
+import cn.seecoder.fdroidrepository.Utils.SensitiveWordUtils;
+import cn.seecoder.fdroidrepository.WordCheck.WordCheckHelper;
+import com.github.houbb.heaven.util.lang.StringUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+import java.io.FileWriter;
+import java.io.IOException;
+
+@Service
+public class SensitiveWordServiceImpl implements SensitiveWordService {
+    private static final Logger logger = LoggerFactory.getLogger(SensitiveWordServiceImpl.class);
+
+    @Override
+    public Response list() {
+        return Response.buildSuccess(SensitiveWordUtils.getSensitiveWords());
+    }
+
+    @Override
+    public Response add(String word) {
+        if (StringUtil.isEmpty(word)) {
+            return Response.buildFailure(ResultCode.ILLEGAL_ARGUMENT.getCode(), ResultCode.ILLEGAL_ARGUMENT.getMessage());
+        }
+        // 将敏感词持久化保存，下次应用启动时不会丢失
+        try {
+            FileWriter writer = new FileWriter("/sensitive/dic.txt", true);
+            writer.append(word);
+        } catch (IOException exception) {
+            logger.error("保存敏感词至本地错误，{}", exception.getMessage());
+            return Response.buildFailure(ResultCode.SENSITIVE_ADD_ERROR.getCode(), ResultCode.SENSITIVE_ADD_ERROR.getMessage());
+        }
+        // 将新的敏感词添加至字典树中
+        WordCheckHelper.addSensitiveWord(word);
+        return Response.buildSuccess();
+    }
+}

src/main/java/cn/seecoder/fdroidrepository/Utils/SensitiveWordUtils.java

@@ -0,0 +1,19 @@
+package cn.seecoder.fdroidrepository.Utils;
+
+import com.github.houbb.heaven.util.io.StreamUtil;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class SensitiveWordUtils {
+    private static List<String> sensitiveWords = new ArrayList<>();
+
+    static {
+        sensitiveWords.addAll(StreamUtil.readAllLines("/sensitive/dic.txt"));
+        sensitiveWords.addAll(StreamUtil.readAllLines("/sensitive/dic_en.txt"));
+    }
+
+    public static List<String> getSensitiveWords() {
+        return sensitiveWords;
+    }
+}

src/main/java/cn/seecoder/fdroidrepository/WordCheck/WordCheckConfig.java

@@ -1,11 +1,15 @@
 package cn.seecoder.fdroidrepository.WordCheck;
 
+import cn.seecoder.fdroidrepository.Utils.SensitiveWordUtils;
 import cn.seecoder.fdroidrepository.WordCheck.interfaces.IWordCheck;
 import cn.seecoder.fdroidrepository.WordCheck.interfaces.IWordFormat;
+import cn.seecoder.fdroidrepository.WordCheck.interfaces.IWordIgnore;
 import cn.seecoder.fdroidrepository.WordCheck.support.check.WordCheckDefault;
 import cn.seecoder.fdroidrepository.WordCheck.support.data.WordTree;
 import cn.seecoder.fdroidrepository.WordCheck.support.format.WordFormatDefault;
+import cn.seecoder.fdroidrepository.WordCheck.support.ignore.WordIgnoreDefault;
 import com.github.houbb.heaven.util.io.StreamUtil;
+import com.github.houbb.heaven.util.lang.StringUtil;
 import lombok.Data;
 
 import java.util.List;
@@ -17,22 +21,39 @@ import java.util.List;
 public class WordCheckConfig {
     private IWordFormat wordFormat;
     private IWordCheck wordCheck;
+    private IWordIgnore wordIgnore;
     private WordTree wordTree;
+    private WordTree htmlTagTree;
 
     public static WordCheckConfig init() {
         WordCheckConfig instance = new WordCheckConfig();
 
         instance.wordFormat = WordFormatDefault.getInstance();
         instance.wordCheck = WordCheckDefault.getInstance();
+        instance.wordIgnore = WordIgnoreDefault.getInstance();
 
         /**
          * 字典树的初始化
          */
-        List<String> words = StreamUtil.readAllLines("/sensitive/dic.txt");
-        words.addAll(StreamUtil.readAllLines("/sensitive/dic_en.txt"));
+//        List<String> words = StreamUtil.readAllLines("/sensitive/dic.txt");
+//        words.addAll(StreamUtil.readAllLines("/sensitive/dic_en.txt"));
         instance.wordTree = new WordTree();
-        instance.wordTree.init(words);
+        instance.wordTree.init(SensitiveWordUtils.getSensitiveWords());
+
+        /**
+         * Html标签字典树初始化
+         */
+        List<String> tags = StreamUtil.readAllLines("/sensitive/dic_html.txt");
+        instance.htmlTagTree = new WordTree();
+        instance.htmlTagTree.init(tags);
 
         return instance;
     }
+
+    public void addSensitiveWord(String word) {
+        if (StringUtil.isEmpty(word)) {
+            return;
+        }
+        this.wordTree.addNode(word);
+    }
 }

src/main/java/cn/seecoder/fdroidrepository/WordCheck/WordCheckContext.java

@@ -13,4 +13,12 @@ public class WordCheckContext {
     private String originText;
     private Map<Character, Character> formattedCharacterMapping;
     private WordCheckConfig wordCheckConfig;
+    private char[] rawChars;
+
+    public WordCheckContext(String text, Map<Character, Character> map, WordCheckConfig config) {
+        this.originText = text;
+        this.formattedCharacterMapping = map;
+        this.wordCheckConfig = config;
+        this.rawChars = text.toCharArray();
+    }
 }

src/main/java/cn/seecoder/fdroidrepository/WordCheck/WordCheckHelper.java

@@ -8,7 +8,6 @@ import java.util.List;
 /**
  * 敏感词检查工具类，对外暴露接口
  */
-@Component
 public class WordCheckHelper {
 
     /**
@@ -28,8 +27,17 @@ public class WordCheckHelper {
         return checkTool.findAll(text, config);
     }
 
+    /**
+     * 增加新的敏感词
+     * @param word 需要添加的敏感词
+     */
+    public static void addSensitiveWord(String word) {
+        config.addSensitiveWord(word);
+    }
+
     public static void main(String[] args) {
-        List<WordCheckResult> result = findAll("this is a conversation");
+        config.addSensitiveWord("guest");
+        List<WordCheckResult> result = findAll("fuuu<div>cccc</p>kkkkk");
         return;
     }
 }

src/main/java/cn/seecoder/fdroidrepository/WordCheck/interfaces/IWordIgnore.java

@@ -0,0 +1,14 @@
+package cn.seecoder.fdroidrepository.WordCheck.interfaces;
+
+import cn.seecoder.fdroidrepository.WordCheck.WordCheckContext;
+
+public interface IWordIgnore {
+    /**
+     * 判断是否需要忽略 index处及其之后的一些特殊字符
+     * e.g .$&等特殊字符 以及 <div></div>等html标签
+     * @param index 当前位置
+     * @param context 上下文
+     * @return 忽略的字符长度
+     */
+    Integer doIgnore(int index, WordCheckContext context);
+}

src/main/java/cn/seecoder/fdroidrepository/WordCheck/support/check/WordCheckWord.java

@@ -3,6 +3,7 @@ package cn.seecoder.fdroidrepository.WordCheck.support.check;
 import cn.seecoder.fdroidrepository.WordCheck.WordCheckContext;
 import cn.seecoder.fdroidrepository.WordCheck.WordCheckConfig;
 import cn.seecoder.fdroidrepository.WordCheck.interfaces.IWordCheck;
+import cn.seecoder.fdroidrepository.WordCheck.interfaces.IWordIgnore;
 import cn.seecoder.fdroidrepository.WordCheck.result.WordContainTypeEnum;
 import cn.seecoder.fdroidrepository.WordCheck.support.data.WordTree;
 import cn.seecoder.fdroidrepository.WordCheck.utils.SpecialCharacterIgnoreUtils;
@@ -23,23 +24,21 @@ public class WordCheckWord extends AbstractWordCheck {
 
     @Override
     protected int getActualLength(int beginIndex, WordCheckContext checkContext) {
-        final String text = checkContext.getOriginText();
         final Map<Character, Character> formattedCharacterMapping = checkContext.getFormattedCharacterMapping();
         final WordCheckConfig wordCheckConfig = checkContext.getWordCheckConfig();
 
         int actualLength = 0;
         final WordTree tree = wordCheckConfig.getWordTree();
+        final IWordIgnore wordIgnore = wordCheckConfig.getWordIgnore();
 
-        char[] rawChars = text.toCharArray();
+        char[] rawChars = checkContext.getRawChars();
         StringBuilder stringBuilder = new StringBuilder();
         int tempLen = 0;
         for (int i = beginIndex; i < rawChars.length; i++) {
-            if (SpecialCharacterIgnoreUtils.check(rawChars[i])) {
-                /**
-                 * 忽略特殊字符进行匹配
-                 * e.g f.u.c.k 可以匹配到敏感词 fuck
-                 */
-                tempLen ++;
+            int ignoredLength = wordIgnore.doIgnore(i, checkContext);
+            if (ignoredLength > 0) {
+                tempLen += ignoredLength;
+                i += ignoredLength - 1;
                 continue;
             }
 

src/main/java/cn/seecoder/fdroidrepository/WordCheck/support/data/WordTree.java

@@ -39,6 +39,23 @@ public class WordTree {
         this.root = newRoot;
     }
 
+    public synchronized void addNode(String data) {
+        if (StringUtil.isEmpty(data)) return;
+        synchronized (this.root) {
+            WordTreeNode tempNode = this.root;
+            char[] chars = data.toCharArray();
+            for (char c : chars) {
+                WordTreeNode subNode = tempNode.getSubNode(c);
+                if (subNode == null) {
+                    subNode = new WordTreeNode();
+                    tempNode.addSubNode(c, subNode);
+                }
+                tempNode = subNode;
+            }
+            tempNode.end(true);
+        }
+    }
+
     public WordContainTypeEnum contains(StringBuilder stringBuilder) {
         WordTreeNode node = root;
         int length = stringBuilder.length();

src/main/java/cn/seecoder/fdroidrepository/WordCheck/support/ignore/WordIgnoreDefault.java

@@ -0,0 +1,27 @@
+package cn.seecoder.fdroidrepository.WordCheck.support.ignore;
+
+import cn.seecoder.fdroidrepository.WordCheck.WordCheckContext;
+import cn.seecoder.fdroidrepository.WordCheck.interfaces.IWordIgnore;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class WordIgnoreDefault implements IWordIgnore {
+    private static final IWordIgnore INSTANCE = new WordIgnoreDefault();
+    public static IWordIgnore getInstance() {
+        return INSTANCE;
+    }
+
+    private static final List<IWordIgnore> chains = Arrays.asList(WordIgnoreHtmlTags.getInstance(), WordIgnoreSpecialChars.getInstance());
+
+    @Override
+    public Integer doIgnore(int index, WordCheckContext context) {
+        for (IWordIgnore chain : chains) {
+            int ret = chain.doIgnore(index, context);
+            if (ret > 0) {
+                return ret;
+            }
+        }
+        return 0;
+    }
+}

src/main/java/cn/seecoder/fdroidrepository/WordCheck/support/ignore/WordIgnoreHtmlTags.java

@@ -0,0 +1,39 @@
+package cn.seecoder.fdroidrepository.WordCheck.support.ignore;
+
+import cn.seecoder.fdroidrepository.WordCheck.WordCheckContext;
+import cn.seecoder.fdroidrepository.WordCheck.interfaces.IWordIgnore;
+import cn.seecoder.fdroidrepository.WordCheck.result.WordContainTypeEnum;
+import cn.seecoder.fdroidrepository.WordCheck.support.data.WordTree;
+
+public class WordIgnoreHtmlTags implements IWordIgnore {
+    private static final IWordIgnore INSTANCE = new WordIgnoreHtmlTags();
+    public static IWordIgnore getInstance() {
+        return INSTANCE;
+    }
+
+    @Override
+    public Integer doIgnore(int index, WordCheckContext context) {
+        final char[] rawChars = context.getRawChars();
+        final WordTree tagTree = context.getWordCheckConfig().getHtmlTagTree();
+
+        StringBuilder stringBuilder = new StringBuilder();
+
+        Integer ignoreLength = 0;
+        Integer tempLength = 0;
+        for (int i = index; i < rawChars.length; i++) {
+            stringBuilder.append(rawChars[i]);
+            tempLength ++;
+
+            WordContainTypeEnum result = tagTree.contains(stringBuilder);
+
+            if (WordContainTypeEnum.CONTAINS_END.equals(result)) {
+                ignoreLength = tempLength;
+            }
+
+            if (WordContainTypeEnum.NOT_FOUND.equals(result)) {
+                break;
+            }
+        }
+        return ignoreLength;
+    }
+}

src/main/java/cn/seecoder/fdroidrepository/WordCheck/support/ignore/WordIgnoreSpecialChars.java

@@ -0,0 +1,28 @@
+package cn.seecoder.fdroidrepository.WordCheck.support.ignore;
+
+import cn.seecoder.fdroidrepository.WordCheck.WordCheckContext;
+import cn.seecoder.fdroidrepository.WordCheck.interfaces.IWordIgnore;
+import com.github.houbb.heaven.util.lang.StringUtil;
+
+import java.util.Set;
+
+public class WordIgnoreSpecialChars implements IWordIgnore {
+    private static final IWordIgnore INSTANCE = new WordIgnoreSpecialChars();
+    public static IWordIgnore getInstance() {
+        return INSTANCE;
+    }
+
+    private static final String SPECIAL = "`-=~!@#$%^&*()_+[]{}\\|;:'\",./<>?";
+
+    private static final Set<Character> SET;
+
+    static {
+        SET = StringUtil.toCharSet(SPECIAL);
+    }
+    @Override
+    public Integer doIgnore(int index, WordCheckContext context) {
+        char c = context.getOriginText().charAt(index);
+
+        return SET.contains(c) ? 1 : 0;
+    }
+}

src/main/resources/sensitive/dic_html.txt

@@ -0,0 +1,24 @@
+<div>
+</div>
+<p>
+</p>
+<i>
+</i>
+<br>
+</br>
+<link>
+</link>
+<img>
+</img>
+<h1>
+</h1>
+<h2>
+</h2>
+<h3>
+</h3>
+<h4>
+</h4>
+<h5>
+</h5>
+<h6>
+</h6>