Merge pull request #376 from jplag/frontend-ext-prep

Prepare frontend utilites to make adding more frontends easier

jplag.frontend-utils/src/main/java/de/jplag/AbstractParser.java

@@ -1,30 +1,34 @@
 package de.jplag;
 
 /**
+ * Abstract parser class. Counts errors and manages an error consumer.
  * @author Emeric Kwemou
  */
 public abstract class AbstractParser {
     protected ErrorConsumer errorConsumer;
     protected int errors = 0;
-    private int numberOfErrors = 0;
 
-    public boolean hasErrors() {
-        return errors != 0;
-    }
-
-    public int errorsCount() {
-        return numberOfErrors;
+    public AbstractParser(ErrorConsumer errorConsumer) {
+        this.errorConsumer = errorConsumer;
     }
 
-    protected void parseEnd() {
-        numberOfErrors += errors;
+    /**
+     * @return true if the last parse call (which could be still ongoing) lead to one or more errors.
+     */
+    public boolean hasErrors() {
+        return errors != 0;
     }
 
+    /**
+     * @return the error consumer that collects and prints errors.
+     */
     public ErrorConsumer getErrorConsumer() {
         return errorConsumer;
     }
 
-    public void setErrorConsumer(ErrorConsumer errorConsumer) {
-        this.errorConsumer = errorConsumer;
-    }
+    /**
+     * Setter for the error consumer that collects and prints errors.
+     * @param errorConsumer is the consumer to set.
+     */
+
 }

jplag.frontend-utils/src/main/java/de/jplag/Language.java

@@ -29,19 +29,17 @@ public interface Language {
 
     /**
      * Parses a set files in a directory.
+     * @param directory is the directory where the files are located.
+     * @param files are the names of the files to parse.
+     * @return the list of parsed JPlag tokens.
      */
-    TokenList parse(File dir, String[] files);
+    TokenList parse(File directory, String[] files);
 
     /**
      * Whether errors were found during the last {@link #parse}.
      */
     boolean hasErrors();
 
-    /**
-     * Number of errors found during the last {@link #parse}.
-     */
-    int errorCount();
-
     /**
      * Determines whether the parser provide column information.
      */

jplag.frontend-utils/src/main/java/de/jplag/Token.java

@@ -13,7 +13,7 @@ public abstract class Token {
 
     private boolean marked;
     private boolean basecode = false;
-    private int hash = -1;// hash-value. set and used by main algorithm (GSTiling)
+    private int hash = -1; // hash-value. set and used by main algorithm (GSTiling)
 
     protected int type;
 
@@ -43,12 +43,19 @@ public abstract class Token {
         this.length = length;
     }
 
-    public String getFile() {
-        return file;
+    /**
+     * Returns the character index which denotes where the code sections represented by this token starts in the line.
+     * @return the character index in the line.
+     */
+    public int getColumn() {
+        return column;
     }
 
-    public int getHash() {
-        return hash;
+    /**
+     * @return the name of the file where the source code that the token represents is located in.
+     */
+    public String getFile() {
+        return file;
     }
 
     // this is made to distinguish the character front end. Maybe other front ends can use it too?
@@ -56,59 +63,88 @@ public abstract class Token {
         return -1;
     }
 
-    public int getLine() {
-        return line;
-    }
-
-    public int getColumn() {
-        return column;
-    }
-
+    /**
+     * Gives the length if the code sections represented by this token.
+     * @return the length in characters.
+     */
     public int getLength() {
         return length;
     }
 
-    public boolean isBasecode() {
-        return basecode;
-    }
-
-    public boolean isMarked() {
-        return marked;
-    }
-
-    public void setLine(int line) {
-        this.line = line;
+    /**
+     * Gives the line index denoting in which line the code sections represented by this token starts.
+     * @return the line index.
+     */
+    public int getLine() {
+        return line;
     }
 
+    /**
+     * Sets the character index which denotes where the code sections represented by this token starts in the line.
+     * @param column is the index in characters to set.
+     */
     public void setColumn(int column) {
         this.column = column;
     }
 
+    /**
+     * Sets the length if the code sections represented by this token.
+     * @param length is the length in characters to set.
+     */
     public void setLength(int length) {
         this.length = length;
     }
 
+    /**
+     * Sets the line index denoting in which line the code sections represented by this token starts.
+     * @param line is the line index to set.
+     */
+    public void setLine(int line) {
+        this.line = line;
+    }
+
     @Override
     public String toString() {
         return type2string();
     }
 
+    /**
+     * @return a string representation depending on the type of the token.
+     */
     protected abstract String type2string();
 
-    /* Package Private */ boolean setBasecode(boolean basecode) {
+    /* package-private */ int getHash() {
+        return hash;
+    }
+
+    /**
+     * @return whether this token is part of a basecode.
+     */
+    /* package-private */ boolean isBasecode() {
+        return basecode;
+    }
+
+    /**
+     * @return whether this token is marked by the comparison algorithm.
+     */
+    /* package-private */ boolean isMarked() {
+        return marked;
+    }
+
+    /* package-private */ boolean setBasecode(boolean basecode) {
         this.basecode = basecode;
         return basecode;
     }
 
-    /* Package Private */ void setFile(String file) {
+    /* package-private */ void setFile(String file) {
         this.file = file;
     }
 
-    /* Package Private */ void setHash(int hash) {
+    /* package-private */ void setHash(int hash) {
         this.hash = hash;
     }
 
-    /* Package Private */ boolean setMarked(boolean marked) {
+    /* package-private */ boolean setMarked(boolean marked) {
         this.marked = marked;
         return marked;
     }

jplag.frontend-utils/src/main/java/de/jplag/TokenConstants.java

@@ -10,7 +10,7 @@ public interface TokenConstants {
     int FILE_END = 0;
 
     /**
-     * Used to optionally separate methods from each other with an always marked token
+     * Used to optionally separate methods from each other with an always marked token.
      */
     int SEPARATOR_TOKEN = 1;
 }

jplag.frontend-utils/src/main/java/de/jplag/TokenHashMap.java

@@ -13,6 +13,7 @@ import java.util.Map;
  * that is larger or equal to the specified size (see {@link TokenHashMap#TokenHashMap(int)}).
  */
 public class TokenHashMap {
+    private static final int CERTAINTY = 100;
     private final Map<Integer, List<Integer>> mappedEntries;
     private final int primeNumber;
 
@@ -67,7 +68,7 @@ public class TokenHashMap {
         }
         for (int possiblePrime = number; possiblePrime < 2 * number; possiblePrime++) { // Bertrand's postulate
             BigInteger bigInt = BigInteger.valueOf(possiblePrime);
-            if (bigInt.isProbablePrime(100)) {
+            if (bigInt.isProbablePrime(CERTAINTY)) {
                 return possiblePrime;
             }
         }

jplag.frontend-utils/src/main/java/de/jplag/TokenList.java

@@ -1,5 +1,7 @@
 package de.jplag;
 
+import static java.util.stream.Collectors.toList;
+
 import java.util.ArrayList;
 import java.util.List;
 
@@ -9,16 +11,26 @@ import java.util.List;
 public class TokenList {
     private final List<Token> tokens;
     TokenHashMap tokenHashes = null;
-    int hash_length = -1;
+    int hashLength = -1;
 
+    /**
+     * Creates an empty token list.
+     */
     public TokenList() {
         tokens = new ArrayList<>();
     }
 
+    /**
+     * @return the number of tokens in the list.
+     */
     public final int size() {
         return tokens.size();
     }
 
+    /**
+     * Adds an token to the list.
+     * @param token is the token to add.
+     */
     public final void addToken(Token token) {
         if (tokens.size() > 0) {
             Token lastToken = tokens.get(tokens.size() - 1);
@@ -55,19 +67,11 @@ public class TokenList {
 
     @Override
     public final String toString() {
-        StringBuilder buffer = new StringBuilder();
         try {
-            for (int i = 0; i < tokens.size(); i++) {
-                buffer.append(i);
-                buffer.append("\t");
-                buffer.append(tokens.get(i).toString());
-                if (i < tokens.size() - 1) {
-                    buffer.append("\n");
-                }
-            }
-        } catch (OutOfMemoryError e) {
-            return "Tokenlist to large for output: " + tokens.size() + " Tokens";
+            List<String> tokenStrings = tokens.stream().map(Token::toString).collect(toList());
+            return String.join(System.lineSeparator(), tokenStrings);
+        } catch (OutOfMemoryError exception) {
+            return "Token list to large for output: " + tokens.size() + " Tokens";
         }
-        return buffer.toString();
     }
 }

jplag.frontend.chars/src/main/java/de/jplag/chars/Language.java

@@ -10,10 +10,10 @@ import de.jplag.TokenList;
  */
 public class Language implements de.jplag.Language {
 
-    private final de.jplag.chars.Parser parser = new Parser();
+    private final Parser parser;
 
     public Language(ErrorConsumer program) {
-        this.parser.setErrorConsumer(program);
+        parser = new Parser(program);
     }
 
     @Override
@@ -21,11 +21,6 @@ public class Language implements de.jplag.Language {
         return new String[] {".TXT", ".txt", ".ASC", ".asc", ".TEX", ".tex"};
     }
 
-    @Override
-    public int errorCount() {
-        return this.parser.errorsCount();
-    }
-
     @Override
     public String getName() {
         return "Character Parser";

jplag.frontend.chars/src/main/java/de/jplag/chars/Parser.java

@@ -5,6 +5,7 @@ import java.io.FileReader;
 import java.nio.charset.StandardCharsets;
 
 import de.jplag.AbstractParser;
+import de.jplag.ErrorConsumer;
 import de.jplag.TokenConstants;
 import de.jplag.TokenList;
 
@@ -12,6 +13,14 @@ public class Parser extends AbstractParser {
 
     private TokenList tokens;
 
+    /**
+     * Creates the parser.
+     * @param errorConsumer is the consumer for any occurring errors.
+     */
+    public Parser(ErrorConsumer errorConsumer) {
+        super(errorConsumer);
+    }
+
     public TokenList parse(File directory, String[] files) {
         tokens = new TokenList();
         errors = 0;
@@ -26,7 +35,6 @@ public class Parser extends AbstractParser {
         else
             errorConsumer.print(null, errors + " ERROR" + (errors > 1 ? "S" : ""));
 
-        this.parseEnd();
         return tokens;
     }
 

jplag.frontend.cpp/src/main/java/de/jplag/cpp/Language.java

@@ -8,14 +8,8 @@ import de.jplag.TokenList;
 public class Language implements de.jplag.Language {
     private final Scanner scanner; // cpp code is scanned not parsed
 
-    public Language(ErrorConsumer program) {
-        this.scanner = new Scanner();
-        this.scanner.setErrorConsumer(program);
-    }
-
-    @Override
-    public int errorCount() {
-        return this.scanner.errorsCount();
+    public Language(ErrorConsumer errorConsumer) {
+        scanner = new Scanner(errorConsumer);
     }
 
     @Override

jplag.frontend.cpp/src/main/java/de/jplag/cpp/Scanner.java

@@ -3,6 +3,7 @@ package de.jplag.cpp;
 import java.io.File;
 
 import de.jplag.AbstractParser;
+import de.jplag.ErrorConsumer;
 import de.jplag.TokenList;
 
 public class Scanner extends AbstractParser {
@@ -10,6 +11,14 @@ public class Scanner extends AbstractParser {
 
     private TokenList tokens;
 
+    /**
+     * Creates the parser.
+     * @param errorConsumer is the consumer for any occurring errors.
+     */
+    public Scanner(ErrorConsumer errorConsumer) {
+        super(errorConsumer);
+    }
+
     public TokenList scan(File directory, String[] files) {
         tokens = new TokenList();
         errors = 0;
@@ -21,7 +30,6 @@ public class Scanner extends AbstractParser {
             }
             tokens.addToken(new CPPToken(CPPTokenConstants.FILE_END, currentFile));
         }
-        this.parseEnd();
         return tokens;
     }
 

jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/Language.java

@@ -9,9 +9,7 @@ public class Language implements de.jplag.Language {
     private final Parser parser;
 
     public Language(ErrorConsumer program) {
-        this.parser = new Parser();
-        this.parser.setErrorConsumer(program);
-
+        parser = new Parser(program);
     }
 
     @Override
@@ -19,11 +17,6 @@ public class Language implements de.jplag.Language {
         return new String[] {".cs", ".CS"};
     }
 
-    @Override
-    public int errorCount() {
-        return this.parser.errorsCount();
-    }
-
     @Override
     public String getName() {
         return "C# 1.2 Parser";

jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/Parser.java

@@ -7,6 +7,7 @@ import java.nio.charset.StandardCharsets;
 import antlr.Token;
 
 import de.jplag.AbstractParser;
+import de.jplag.ErrorConsumer;
 import de.jplag.TokenList;
 import de.jplag.csharp.grammar.CSharpLexer;
 import de.jplag.csharp.grammar.CSharpParser;
@@ -15,6 +16,14 @@ public class Parser extends AbstractParser {
     private TokenList tokens;
     private String currentFile;
 
+    /**
+     * Creates the parser.
+     * @param errorConsumer is the consumer for any occurring errors.
+     */
+    public Parser(ErrorConsumer errorConsumer) {
+        super(errorConsumer);
+    }
+
     public TokenList parse(File directory, String[] files) {
         tokens = new TokenList();
         errors = 0;
@@ -23,7 +32,6 @@ public class Parser extends AbstractParser {
                 errors++;
             tokens.addToken(new CSharpToken(CSharpTokenConstants.FILE_END, files[i], -1, -1, -1));
         }
-        this.parseEnd();
         return tokens;
     }
 

jplag.frontend.java/src/main/java/de/jplag/java/Language.java

@@ -12,8 +12,7 @@ public class Language implements de.jplag.Language {
     private final Parser parser;
 
     public Language(ErrorConsumer errorConsumer) {
-        this.parser = new Parser();
-        this.parser.setErrorConsumer(errorConsumer);
+        parser = new Parser(errorConsumer);
     }
 
     @Override
@@ -66,9 +65,4 @@ public class Language implements de.jplag.Language {
         return this.parser.hasErrors();
     }
 
-    @Override
-    public int errorCount() {
-        return this.parser.errorsCount();
-    }
-
 }

jplag.frontend.java/src/main/java/de/jplag/java/Parser.java

@@ -6,17 +6,25 @@ import java.io.File;
 import java.util.Arrays;
 
 import de.jplag.AbstractParser;
+import de.jplag.ErrorConsumer;
 import de.jplag.TokenList;
 
 public class Parser extends AbstractParser {
     private TokenList tokens;
 
+    /**
+     * Creates the parser.
+     * @param errorConsumer is the consumer for any occurring errors.
+     */
+    public Parser(ErrorConsumer errorConsumer) {
+        super(errorConsumer);
+    }
+
     public TokenList parse(File directory, String[] files) {
         tokens = new TokenList();
         errors = 0;
         var pathedFiles = Arrays.stream(files).map(it -> new File(directory, it)).collect(toList());
         errors += new JavacAdapter().parseFiles(directory, pathedFiles, this);
-        this.parseEnd();
         return tokens;
     }
 

jplag.frontend.python-3/src/main/java/de/jplag/python3/Language.java

@@ -9,9 +9,8 @@ public class Language implements de.jplag.Language {
 
     private final Parser parser;
 
-    public Language(ErrorConsumer program) {
-        this.parser = new Parser();
-        this.parser.setErrorConsumer(program);
+    public Language(ErrorConsumer errorConsumer) {
+        parser = new Parser(errorConsumer);
     }
 
     @Override
@@ -19,11 +18,6 @@ public class Language implements de.jplag.Language {
         return new String[] {".py"};
     }
 
-    @Override
-    public int errorCount() {
-        return this.parser.errorsCount();
-    }
-
     @Override
     public String getName() {
         return "Python3 Parser";

jplag.frontend.python-3/src/main/java/de/jplag/python3/Parser.java

@@ -13,6 +13,7 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
+import de.jplag.ErrorConsumer;
 import de.jplag.TokenList;
 import de.jplag.python3.grammar.Python3Lexer;
 import de.jplag.python3.grammar.Python3Parser;
@@ -23,6 +24,14 @@ public class Parser extends AbstractParser {
     private TokenList tokens = new TokenList();
     private String currentFile;
 
+    /**
+     * Creates the parser.
+     * @param errorConsumer is the consumer for any occurring errors.
+     */
+    public Parser(ErrorConsumer errorConsumer) {
+        super(errorConsumer);
+    }
+
     public TokenList parse(File directory, String[] files) {
         tokens = new TokenList();
         errors = 0;
@@ -33,7 +42,6 @@ public class Parser extends AbstractParser {
             }
             tokens.addToken(new Python3Token(Python3TokenConstants.FILE_END, files[i], -1, -1, -1));
         }
-        this.parseEnd();
         return tokens;
     }
 

jplag.frontend.scheme/src/main/java/de/jplag/scheme/Language.java

@@ -6,20 +6,12 @@ import de.jplag.ErrorConsumer;
 import de.jplag.TokenList;
 
 public class Language implements de.jplag.Language {
+    private final de.jplag.scheme.Parser parser;
 
     public Language(ErrorConsumer program) {
-        this.parser = new Parser();
-        this.parser.setErrorConsumer(program);
-
+        parser = new Parser(program);
     }
 
-    @Override
-    public int errorCount() {
-        return this.parser.errorsCount();
-    }
-
-    private final de.jplag.scheme.Parser parser; // Not yet instantiated? See constructor!
-
     @Override
     public String[] suffixes() {
         return new String[] {".scm", ".SCM", ".ss", ".SS"};

jplag.frontend.scheme/src/main/java/de/jplag/scheme/Parser.java

@@ -3,6 +3,7 @@ package de.jplag.scheme;
 import java.io.File;
 
 import de.jplag.AbstractParser;
+import de.jplag.ErrorConsumer;
 import de.jplag.TokenList;
 
 public class Parser extends AbstractParser {
@@ -10,6 +11,14 @@ public class Parser extends AbstractParser {
 
     private TokenList tokens;
 
+    /**
+     * Creates the parser.
+     * @param errorConsumer is the consumer for any occurring errors.
+     */
+    public Parser(ErrorConsumer errorConsumer) {
+        super(errorConsumer);
+    }
+
     public TokenList parse(File directory, String[] files) {
         tokens = new TokenList();
         errors = 0;
@@ -20,7 +29,6 @@ public class Parser extends AbstractParser {
                 errors++;
             tokens.addToken(new SchemeToken(SchemeTokenConstants.FILE_END, currentFile));
         }
-        this.parseEnd();
         return tokens;
     }
 

jplag.frontend.text/src/main/java/de/jplag/text/Language.java

@@ -8,15 +8,10 @@ import de.jplag.TokenList;
 
 public class Language implements de.jplag.Language {
 
-    private final Parser parser = new Parser();
+    private final Parser parser;
 
-    public Language(ErrorConsumer program) {
-        this.parser.setErrorConsumer(program);
-    }
-
-    @Override
-    public int errorCount() {
-        return this.parser.errorsCount();
+    public Language(ErrorConsumer errorConsumer) {
+        parser = new Parser(errorConsumer);
     }
 
     @Override

jplag.frontend.text/src/main/java/de/jplag/text/Parser.java

@@ -7,6 +7,7 @@ import java.util.Hashtable;
 import antlr.Token;
 
 import de.jplag.AbstractParser;
+import de.jplag.ErrorConsumer;
 import de.jplag.TokenConstants;
 import de.jplag.TokenList;
 
@@ -21,6 +22,14 @@ public class Parser extends AbstractParser {
 
     private boolean runOut = false;
 
+    /**
+     * Creates the parser.
+     * @param errorConsumer is the consumer for any occurring errors.
+     */
+    public Parser(ErrorConsumer errorConsumer) {
+        super(errorConsumer);
+    }
+
     public TokenList parse(File directory, String[] files) {
         tokens = new TokenList();
         errors = 0;
@@ -33,7 +42,6 @@ public class Parser extends AbstractParser {
 
         TokenList tmp = tokens;
         tokens = null;
-        this.parseEnd();
         return tmp;
     }
 

jplag/src/main/java/de/jplag/GreedyStringTiling.java

@@ -25,7 +25,7 @@ public class GreedyStringTiling {
     }
 
     /**
-     * Creating hashes in linear time. The hash-code will be written in every Token for the next <hash_length> token
+     * Creating hashes in linear time. The hash-code will be written in every Token for the next <hashLength> token
      * (includes the Token itself).
      * @param tokenList contains the tokens.
      * @param hashLength is the hash length (condition: 1 < hashLength < 26)
@@ -86,7 +86,7 @@ public class GreedyStringTiling {
                 }
             }
         }
-        tokenList.hash_length = hashLength;
+        tokenList.hashLength = hashLength;
     }
 
     public final JPlagComparison compare(Submission firstSubmission, Submission secondSubmission) {
@@ -139,10 +139,10 @@ public class GreedyStringTiling {
         markTokens(second, isBaseCodeComparison);
 
         // create hashes:
-        if (first.hash_length != minimumTokenMatch) {
+        if (first.hashLength != minimumTokenMatch) {
             createHashes(first, minimumTokenMatch, isBaseCodeComparison); // don't make table if it is not a base code comparison
         }
-        if (second.hash_length != minimumTokenMatch || second.tokenHashes == null) {
+        if (second.hashLength != minimumTokenMatch || second.tokenHashes == null) {
             createHashes(second, minimumTokenMatch, true);
         }