replace ANTLR with CoreNLP

jplag.frontend.text/pom.xml

@@ -9,34 +9,14 @@
     <artifactId>text</artifactId>
 
     <dependencies>
-        <dependency>
-            <groupId>antlr</groupId>
-            <artifactId>antlr</artifactId>
-        </dependency>
         <dependency>
             <groupId>de.jplag</groupId>
             <artifactId>frontend-utils</artifactId>
         </dependency>
+        <dependency>
+            <groupId>edu.stanford.nlp</groupId>
+            <artifactId>stanford-corenlp</artifactId>
+            <version>4.4.0</version>
+        </dependency>
     </dependencies>
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>antlr-maven-plugin</artifactId>
-                <configuration>
-                    <!-- Comma separated list of grammar files or pattern grammar files 
-						By default, grammar file(s) is in ${basedir}/src/main/antlr -->
-                    <!-- <grammars>*.g</grammars> -->
-                    <grammars>text.g</grammars>
-                </configuration>
-                <executions>
-                    <execution>
-                        <goals>
-                            <goal>generate</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
 </project>

jplag.frontend.text/src/main/antlr/text.g

@@ -1,102 +0,0 @@
-header {
-package de.jplag.text;
-}
-
-// tell ANTLR that we want to generate Java source code
-options {
-  language="Java";
-}
-
-class TextParser extends Parser;
-options {
-  k = 2;			  // two token lookahead
-  //  exportVocab=Text;	          // Call its vocabulary "Text"
-  codeGenMakeSwitchThreshold = 2; // Some optimizations
-  codeGenBitsetTestThreshold = 3;
-  defaultErrorHandler = true;
-  buildAST = false;
-  ASTLabelType = "AntlrParserToken";
-}
-
-{
-    private de.jplag.text.ParserAdapter parser;
-
-    public void setParserAdapter(de.jplag.text.ParserAdapter adapter) {
-        this.parser = adapter;
-    }
-}
-
-file : ( w:WORD { parser.add(w); } | PUNCTUATION | SPECIALS )* EOF ;
-
-//----------------------------------------------------------------------------
-// The Text scanner
-//----------------------------------------------------------------------------
-
-{
-import de.jplag.text.InputState;
-import de.jplag.text.AntlrParserToken;
-}
-
-class TextLexer extends Lexer;
-options {
-    //  exportVocab=Text;    // call the vocabulary "Text"
-    testLiterals = false;    // don't automatically test for literals
-    k = 2;                   // two characters of lookahead
-    charVocabulary = '\u0000'..'\u00FF';
-}
-
-{
-    public void newline() {
-        super.newline();
-        ((InputState) inputState).setColumnIndex(1);
-    }
-
-    public void consume() throws antlr.CharStreamException {
-        if (inputState.guessing == 0) {
-            InputState state = (InputState) inputState;
-            if (text.length() == 0) {
-                // remember token start column
-                state.setTokenColumnIndex(state.getColumnIndex());
-            }
-            state.setColumnIndex(state.getColumnIndex() + 1);
-        }
-        super.consume();
-    }
-
-    protected Token makeToken(int t) {
-        AntlrParserToken token = (AntlrParserToken) super.makeToken(t);
-        token.setColumn(((InputState) inputState).getTokenColumnIndex());
-        return token;
-    }
-}
-
-WORD
-options { paraphrase = "an identifier"; } :
-  (( '0'..'9') | ('A'..'Z') | ('a'..'z') |
-   ('\300' .. '\326') | ('\330' .. '\366') | ('\370' .. '\377'))+ ;
-
-PUNCTUATION : (	'!' | '"' | '\'' | '(' | ')' | ',' | '-' | '.' |
-		':' | ';' | '?'  | '[' | ']' | '`' | '{' | '}' |
-		'\253' | '\264' | '\273' | '\277' | '\0') ;
-
-SPECIALS : ('#' | '$' | '%' | '&' | '+' | '<' | '=' | '*' |
-	    '/' | '>' | '@' | '\\' | '^' | '_' | '|' | '~' |
-	    ('\241' .. '\252') | ('\254' .. '\263') | ('\265' .. '\272') |
-	    ('\274' .. '\276') | '\327' | '\367' | ('\200' .. '\237') ) ;
-
-// Whitespace -- ignored
-SPACE : ( ' '
-	  |	'\t' //{ ((InputState)inputState).setColumnIndex((InputState)inputState).getColumnIndex() + 7); }
-	  |	'\f'
-	  |     '\240'
-	  |     ('\001' .. '\010')
-	  |     ('\016' .. '\037')
-	  |     '\013'
-	  |     '\177'
-	) { _ttype = Token.SKIP; } ;
-
-NEWLINE	: // handle newlines
-  ( "\r\n" | // Evil DOS
-    '\r'   | // Macintosh
-    '\n'   ) // Unix (the right way)
-  { newline(); _ttype = Token.SKIP; } ;

jplag.frontend.text/src/main/java/de/jplag/text/AntlrParserToken.java

@@ -1,79 +0,0 @@
-package de.jplag.text;
-
-import antlr.Token;
-
-/**
- * Token of the ANTLR grammar, needs to be converted into a JPlag token by the parser adapter.
- */
-public class AntlrParserToken extends Token {
-    /**
-     * This variable holds the line number of the current token.
-     */
-    private int line = -1;
-
-    /**
-     * This variable holds the column of the current token in its line.
-     */
-    private int column = -1;
-
-    /**
-     * This variable holds the label of the current token.
-     */
-    private String text = null;
-
-    /**
-     * This variable holds the identifier of the current token.
-     */
-    private int id = -1;
-
-    public AntlrParserToken() {
-        super();
-    }
-
-    @Override
-    public void setLine(int line) {
-        this.line = line;
-    }
-
-    @Override
-    public void setColumn(int column) {
-        this.column = column;
-    }
-
-    public void setID(int id) {
-        this.id = id;
-    }
-
-    @Override
-    public void setText(String text) {
-        this.text = (text != null ? text.intern() : null);
-    }
-
-    @Override
-    public int getColumn() {
-        return column;
-    }
-
-    @Override
-    public int getLine() {
-        return line;
-    }
-
-    @Override
-    public String getText() {
-        return text;
-    }
-
-    public int getID() {
-        return id;
-    }
-
-    public int getLength() {
-        return text.length();
-    }
-
-    @Override
-    public String toString() {
-        return "{\"" + getText() + "\", <" + getType() + ">, " + getLine() + " " + getColumn() + "}";
-    }
-}

jplag.frontend.text/src/main/java/de/jplag/text/InputState.java

@@ -1,49 +0,0 @@
-package de.jplag.text;
-
-import java.io.InputStream;
-import java.io.Reader;
-
-import antlr.InputBuffer;
-import antlr.LexerSharedInputState;
-
-/**
- * This object contains the data associated with an input stream of characters. Multiple lexers share a single
- * LexerSharedInputState to lex the same input stream.
- */
-public class InputState extends LexerSharedInputState {
-    private int columnIndex = 1;
-    private int tokenColumnIndex = 1;
-
-    public InputState(InputBuffer inputBuffer) {
-        super(inputBuffer);
-    }
-
-    public InputState(InputStream inputStream) {
-        super(inputStream);
-    }
-
-    public InputState(Reader inputReader) {
-        super(inputReader);
-    }
-
-    @Override
-    public int getLine() {
-        return line;
-    }
-
-    public int getColumnIndex() {
-        return columnIndex;
-    }
-
-    public void setColumnIndex(int columnIndex) {
-        this.columnIndex = columnIndex;
-    }
-
-    public int getTokenColumnIndex() {
-        return tokenColumnIndex;
-    }
-
-    public void setTokenColumnIndex(int tokenColumnIndex) {
-        this.tokenColumnIndex = tokenColumnIndex;
-    }
-}

jplag.frontend.text/src/main/java/de/jplag/text/ParserAdapter.java

@@ -1,24 +1,38 @@
 package de.jplag.text;
 
 import java.io.File;
-import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
-
-import antlr.Token;
+import java.util.Properties;
 
 import de.jplag.AbstractParser;
 import de.jplag.TokenConstants;
 import de.jplag.TokenList;
+import edu.stanford.nlp.ling.CoreLabel;
+import edu.stanford.nlp.pipeline.CoreDocument;
+import edu.stanford.nlp.pipeline.StanfordCoreNLP;
 
 public class ParserAdapter extends AbstractParser {
 
     private final Map<String, Integer> tokenTypes = new HashMap<>();
+    private final StanfordCoreNLP pipeline;
     private int tokenTypeIndex = 1; // 0 is FILE_END token, SEPARATOR is not used as there are no methods.
 
     private TokenList tokens;
     private String currentFile;
 
+    public ParserAdapter() {
+        Properties properties = new Properties();
+        properties.put("annotators", "tokenize");
+        this.pipeline = new StanfordCoreNLP(properties);
+    }
+
     public TokenList parse(File directory, String[] files) {
         tokens = new TokenList();
         errors = 0;
@@ -32,42 +46,65 @@ public class ParserAdapter extends AbstractParser {
         return tokens;
     }
 
-    /**
-     * Converts a ANTLR token into a JPlag token and adds it to the token list.
-     * @param token is the ANTLR token to convert.
-     */
-    public void add(Token token) {
-        if (token instanceof AntlrParserToken parserToken) {
-            String text = token.getText();
-            int type = getTokenType(text);
-            tokens.addToken(new TextToken(text, type, currentFile, parserToken));
-        } else {
-            throw new IllegalArgumentException("Illegal token implementation: " + token);
+    private boolean parseFile(File directory, String file) {
+        currentFile = file;
+        Path filePath = directory.toPath().resolve(file);
+        ParsedDocument parsed = readDocument(filePath);
+        if (parsed == null) {
+            return false;
         }
+        for (CoreLabel token : parsed.document().tokens()) {
+            if (isWord(token)) {
+                addToken(token, parsed.linebreaks());
+            }
+        }
+        return true;
     }
 
-    private boolean parseFile(File directory, String file) {
-        InputState inputState = null;
-        try (FileInputStream inputStream = new FileInputStream(new File(directory, file))) {
-            currentFile = file;
-            // Create a scanner that reads from the input stream passed to us
-            inputState = new InputState(inputStream);
-            TextLexer lexer = new TextLexer(inputState);
-            lexer.setFilename(file);
-            lexer.setTokenObjectClass("de.jplag.text.AntlrParserToken");
+    private boolean isWord(CoreLabel token) {
+        if (true) {
+            return true;
+        }
+        String text = token.originalText();
+        for (int i = 0; i < text.length(); i++) {
+            char c = text.charAt(i);
+            if (!Character.isAlphabetic(c) || Character.isDigit(c)) {
+                return false;
+            }
+        }
+        return true;
+    }
 
-            // Create a parser that reads from the scanner
-            TextParser parser = new TextParser(lexer);
-            parser.setFilename(file);
-            parser.setParserAdapter(this);
+    private void addToken(CoreLabel label, List<Integer> linebreaks) {
+        String text = label.originalText();
+        int type = getTokenType(text);
+        // assuming we never hit a line break with the beginPosition,
+        // we can directly negate the value to get the line
+        int line = -Collections.binarySearch(linebreaks, label.beginPosition());
+        int column;
+        // line 1 has no start position in the linebreaks list
+        if (line == 1) {
+            column = label.beginPosition();
+        } else {
+            // while token lines start at 1, the list is 0-indexed.
+            // additionally, we need to skip line 1 here => - 2
+            column = label.beginPosition() - linebreaks.get(line - 2);
+        }
+        int length = label.endPosition() - label.beginPosition();
+        TokenPosition position = new TokenPosition(line, column, length);
+        tokens.addToken(new TextToken(text, type, currentFile, position));
+    }
 
-            // start parsing at the compilationUnit rule
-            parser.file();
-        } catch (Exception e) {
-            logger.error("Parsing Error in " + file + " (line " + (inputState != null ? "" + inputState.getLine() : "") + "):" + e.getMessage(), e);
-            return false;
+    private ParsedDocument readDocument(Path filePath) {
+        String content;
+        try {
+            content = Files.readString(filePath);
+        } catch (IOException e) {
+            logger.error("Error reading from file {}", filePath, e);
+            return null;
         }
-        return true;
+        List<Integer> linebreaks = createLinebreaksLookup(content);
+        return new ParsedDocument(pipeline.processToCoreDocument(content), linebreaks);
     }
 
     private int getTokenType(String text) {
@@ -79,6 +116,20 @@ public class ParserAdapter extends AbstractParser {
             return ++tokenTypeIndex;
         });
         return tokenTypes.get(text);
+    }
+
+    // TODO we can probably get that data from CoreNLP?
+    private List<Integer> createLinebreaksLookup(String content) {
+        List<Integer> linebreaks = new ArrayList<>();
+        for (int i = 0; i < content.length(); i++) {
+            if (content.charAt(i) == '\n') {
+                linebreaks.add(i);
+            }
+        }
+        return linebreaks;
+    }
+
+    record ParsedDocument(CoreDocument document, List<Integer> linebreaks) {
 
     }
 }

jplag.frontend.text/src/main/java/de/jplag/text/TextToken.java

@@ -12,8 +12,8 @@ public class TextToken extends Token {
         this.text = NO_TEXT;
     }
 
-    public TextToken(String text, int type, String file, AntlrParserToken parserToken) {
-        super(type, file, parserToken.getLine(), parserToken.getColumn(), parserToken.getLength());
+    public TextToken(String text, int type, String file, TokenPosition parserToken) {
+        super(type, file, parserToken.line(), parserToken.column(), parserToken.length());
         this.text = text.toLowerCase();
     }
 

jplag.frontend.text/src/main/java/de/jplag/text/TokenPosition.java

@@ -0,0 +1,8 @@
+package de.jplag.text;
+
+public record TokenPosition(
+    int line,
+    int column,
+    int length
+) {
+}