Merge pull request #534 from jplag/core-algorithm-refactoring

Core algorithm refactoring

jplag.endtoend-testing/src/main/java/de/jplag/endtoend/helper/TestSuiteHelper.java

@@ -56,7 +56,7 @@ public class TestSuiteHelper {
      */
     public static String getTestIdentifier(JPlagComparison jPlagComparison) {
 
-        return List.of(jPlagComparison.getFirstSubmission(), jPlagComparison.getSecondSubmission()).stream().map(Submission::getFiles)
+        return List.of(jPlagComparison.firstSubmission(), jPlagComparison.secondSubmission()).stream().map(Submission::getFiles)
                 .map(FileHelper::getEnclosedFileNamesFromCollection).sorted().collect(Collectors.joining("-"));
 
     }

jplag.frontend-testutils/src/test/java/de/jplag/testutils/TokenUtils.java

@@ -3,7 +3,6 @@ package de.jplag.testutils;
 import java.util.List;
 
 import de.jplag.Token;
-import de.jplag.TokenList;
 
 public final class TokenUtils {
 
@@ -12,25 +11,23 @@ public final class TokenUtils {
     }
 
     /**
-     * Returns the type of all tokens in a {@link TokenList} that belong to a file.
-     * @param tokenList is the {@link TokenList}.
+     * Returns the type of all tokens that belong to a certain file.
+     * @param tokenList is the list of {@link Token Tokens}.
      * @param name is the name of the target file.
      * @return the immutable list of token types.
      */
-    public static List<Integer> tokenTypesByFile(TokenList tokenList, String name) {
-        var tokens = tokensByFile(tokenList, name);
-        return tokens.stream().map(Token::getType).toList();
+    public static List<Integer> tokenTypesByFile(List<Token> tokens, String name) {
+        return tokensByFile(tokens, name).stream().map(Token::getType).toList();
     }
 
     /**
-     * Returns the tokens in a {@link TokenList} that belong to a file.
-     * @param tokenList is the {@link TokenList}.
+     * Returns the tokens that belong to a certain file.
+     * @param tokenList is the list of {@link Token Tokens}.
      * @param name is the name of the target file.
      * @return the immutable list of tokens.
      */
-    public static List<Token> tokensByFile(TokenList tokenList, String name) {
-        var tokens = tokenList.allTokens().stream();
-        return tokens.filter(it -> it.getFile().startsWith(name)).toList();
+    public static List<Token> tokensByFile(List<Token> tokens, String name) {
+        return tokens.stream().filter(it -> it.getFile().startsWith(name)).toList();
     }
 
 }

jplag.frontend-utils/src/main/java/de/jplag/Language.java

@@ -1,6 +1,7 @@
 package de.jplag;
 
 import java.io.File;
+import java.util.List;
 
 /**
  * Common interface for all languages. Each language-front end must provide a concrete language implementation.
@@ -33,7 +34,7 @@ public interface Language {
      * @param files are the names of the files to parse.
      * @return the list of parsed JPlag tokens.
      */
-    TokenList parse(File directory, String[] files);
+    List<Token> parse(File directory, String[] files);
 
     /**
      * Whether errors were found during the last {@link #parse}.

jplag.frontend-utils/src/main/java/de/jplag/Token.java

@@ -1,32 +1,36 @@
 package de.jplag;
 
 /**
- * This class represents a token in a source code. It can represents keywords, identifies, syntactical structures etc.
+ * This class represents a token in a source code. It can represent keywords, identifiers, syntactical structures etc.
  * What types of tokens there are depends on the specific language, meaning JPlag does not enforce a specific token set.
  * The language parsers decide what is a token and what is not.
  */
 public abstract class Token {
+    /** Indicates that the requested field has no value. */
+    public static final int NO_VALUE = -1;
+
     private int line;
     private int column;
     private int length;
     private String file;
 
-    private boolean marked;
-    private boolean basecode = false;
-    private int hash = -1; // hash-value. set and used by main algorithm (GSTiling)
-
     protected int type;
 
     /**
      * Creates a token without information about the column or the length of the token in the line.
      * @param type is the token type.
      * @param file is the name of the source code file.
-     * @param line is the line index in the source code where the token resides. Cannot be smaller than 1.
+     * @param line is the line index in the source code where the token resides. Cannot be smaller than 1. For
+     * {@link TokenConstants#FILE_END FILE_END} it is automatically set to {@link #NO_VALUE}.
      */
     public Token(int type, String file, int line) {
         this.type = type;
         this.file = file;
-        setLine(line > 0 ? line : 1);
+        if (type == TokenConstants.FILE_END) {
+            this.line = NO_VALUE;
+        } else {
+            this.line = line > 0 ? line : 1;
+        }
     }
 
     /**
@@ -89,22 +93,6 @@ public abstract class Token {
         this.column = column;
     }
 
-    /**
-     * Sets the length if the code sections represented by this token.
-     * @param length is the length in characters to set.
-     */
-    public void setLength(int length) {
-        this.length = length;
-    }
-
-    /**
-     * Sets the line index denoting in which line the code sections represented by this token starts.
-     * @param line is the line index to set.
-     */
-    public void setLine(int line) {
-        this.line = line;
-    }
-
     @Override
     public String toString() {
         return type2string();
@@ -114,40 +102,4 @@ public abstract class Token {
      * @return a string representation depending on the type of the token.
      */
     protected abstract String type2string();
-
-    /* package-private */ int getHash() {
-        return hash;
-    }
-
-    /**
-     * @return whether this token is part of a basecode.
-     */
-    /* package-private */ boolean isBasecode() {
-        return basecode;
-    }
-
-    /**
-     * @return whether this token is marked by the comparison algorithm.
-     */
-    /* package-private */ boolean isMarked() {
-        return marked;
-    }
-
-    /* package-private */ boolean setBasecode(boolean basecode) {
-        this.basecode = basecode;
-        return basecode;
-    }
-
-    /* package-private */ void setFile(String file) {
-        this.file = file;
-    }
-
-    /* package-private */ void setHash(int hash) {
-        this.hash = hash;
-    }
-
-    /* package-private */ boolean setMarked(boolean marked) {
-        this.marked = marked;
-        return marked;
-    }
 }

jplag.frontend-utils/src/main/java/de/jplag/TokenHashMap.java

@@ -1,77 +0,0 @@
-package de.jplag;
-
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A {@link HashMap} that maps Integer keys to multiple Integer values. Note that all keys with identical
- * <code>(key % prime)</code> are mapped to the same values. Specifically, <code>prime</code> is the next prime number
- * that is larger or equal to the specified size (see {@link TokenHashMap#TokenHashMap(int)}).
- */
-public class TokenHashMap {
-    private static final int CERTAINTY = 100;
-    private final Map<Integer, List<Integer>> mappedEntries;
-    private final int primeNumber;
-
-    /**
-     * Creates the {@link HashMap}.
-     * @param size specifies the initial size and the key mapping (see {@link TokenHashMap}).
-     */
-    public TokenHashMap(int size) {
-        mappedEntries = new HashMap<>(size);
-        primeNumber = nextPrimeNumber(size);
-    }
-
-    /**
-     * Returns all stored numbers for a key. Note that all keys with identical <code>(key % prime)</code> are mapped to the
-     * same values (see {@link TokenHashMap}).
-     * @param key is the specific key.
-     * @return the stored numbers or an empty list if nothing is stored.
-     */
-    public final List<Integer> get(int key) {
-        int actualKey = key % primeNumber;
-        if (mappedEntries.containsKey(actualKey)) {
-            return new ArrayList<>(mappedEntries.get(actualKey));
-        }
-        return Collections.emptyList();
-    }
-
-    /**
-     * Stores a number for a key, does not replace the previous stored numbers for that key. Note that all keys with
-     * identical <code>(key % prime)</code> are mapped to the same values (see {@link TokenHashMap}).
-     * @param key is the specific key.
-     * @param value is the number to store.
-     */
-    public final void put(int key, int value) {
-        int actualKey = key % primeNumber;
-        if (mappedEntries.containsKey(actualKey)) {
-            mappedEntries.get(actualKey).add(value);
-        } else {
-            List<Integer> entries = new ArrayList<>();
-            entries.add(value);
-            mappedEntries.put(actualKey, entries);
-        }
-    }
-
-    /**
-     * Calculates the next prime number (including 1) that is larger or equal to a given number.
-     * @param number is the give number.
-     * @return the next prime number.
-     */
-    private int nextPrimeNumber(int number) {
-        if (number <= 1) {
-            return 1;
-        }
-        for (int possiblePrime = number; possiblePrime < 2 * number; possiblePrime++) { // Bertrand's postulate
-            BigInteger bigInt = BigInteger.valueOf(possiblePrime);
-            if (bigInt.isProbablePrime(CERTAINTY)) {
-                return possiblePrime;
-            }
-        }
-        throw new IllegalStateException("Should never be reached because of Bertrand's postulate!");
-    }
-}

jplag.frontend-utils/src/main/java/de/jplag/TokenList.java

@@ -1,76 +0,0 @@
-package de.jplag;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-/**
- * List of tokens. Allows random access to individual tokens. Contains a hash map for token hashes.
- */
-public class TokenList {
-    private final List<Token> tokens;
-    TokenHashMap tokenHashes = null;
-    int hashLength = -1;
-
-    /**
-     * Creates an empty token list.
-     */
-    public TokenList() {
-        tokens = new ArrayList<>();
-    }
-
-    /**
-     * @return the number of tokens in the list.
-     */
-    public final int size() {
-        return tokens.size();
-    }
-
-    /**
-     * Adds an token to the list.
-     * @param token is the token to add.
-     */
-    public final void addToken(Token token) {
-        if (tokens.size() > 0) {
-            Token lastToken = tokens.get(tokens.size() - 1);
-            if (lastToken.getFile().equals(token.getFile())) {
-                token.setFile(lastToken.getFile()); // To save memory ...
-            }
-            if (token.getLine() < lastToken.getLine() && (token.getFile().equals(lastToken.getFile()))) {
-                token.setLine(lastToken.getLine()); // just to make sure
-            }
-        }
-        tokens.add(token);
-    }
-
-    /**
-     * Returns a view on all tokens.
-     * @return all tokens.
-     */
-    public Collection<Token> allTokens() {
-        return new ArrayList<>(tokens);
-    }
-
-    /**
-     * Grants access to a specific token.
-     * @param index is the token index.
-     * @return the desired token.
-     * @throws IllegalArgumentException if the index is out of bounds.
-     */
-    public Token getToken(int index) {
-        if (index < 0 || index >= tokens.size()) {
-            throw new IllegalArgumentException("Cannot access token with index " + index + ", there are only " + tokens.size() + " tokens!");
-        }
-        return tokens.get(index);
-    }
-
-    @Override
-    public final String toString() {
-        try {
-            List<String> tokenStrings = tokens.stream().map(Token::toString).toList();
-            return String.join(System.lineSeparator(), tokenStrings);
-        } catch (OutOfMemoryError exception) {
-            return "Token list to large for output: " + tokens.size() + " Tokens";
-        }
-    }
-}

jplag.frontend-utils/src/main/java/de/jplag/TokenPrinter.java

@@ -4,7 +4,9 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -46,22 +48,22 @@ public final class TokenPrinter {
 
     /**
      * Creates a string representation of a set of files line by line and adds the tokens under the lines.
-     * @param tokens is the set of tokens parsed from the files.
+     * @param tokens is the list of tokens parsed from the files.
      * @param rootDirectory is the common rootDirectory of the files.
      * @return the string representation.
      */
-    public static String printTokens(TokenList tokens, File rootDirectory) {
+    public static String printTokens(List<Token> tokens, File rootDirectory) {
         return printTokens(tokens, rootDirectory, Optional.empty());
     }
 
     /**
      * Creates a string representation of a collection of files line by line and adds the tokens under the lines.
-     * @param tokenList is the set of tokens parsed from the files.
+     * @param tokenList is the list of tokens parsed from the files.
      * @param rootDirectory is the common directory of the files.
      * @param suffix is the optional view file suffix.
      * @return the string representation.
      */
-    public static String printTokens(TokenList tokenList, File rootDirectory, Optional<String> suffix) {
+    public static String printTokens(List<Token> tokenList, File rootDirectory, Optional<String> suffix) {
         PrinterOutputBuilder builder = new PrinterOutputBuilder();
         Map<String, List<Token>> fileToTokens = groupTokensByFile(tokenList);
 
@@ -124,7 +126,17 @@ public final class TokenPrinter {
         // Sort tokens by file and line -> tokens can be processed without any further checks
         List<String> lines = linesFromFile(file);
 
-        Map<Integer, List<Token>> lineNumbersToTokens = fileTokens.stream().collect(Collectors.groupingBy(Token::getLine));
+        int currentLine = Token.NO_VALUE;
+        Map<Integer, List<Token>> lineNumbersToTokens = new HashMap<>(fileTokens.size());
+        for (Token token : fileTokens) {
+            if (token.getLine() != Token.NO_VALUE) {
+                currentLine = token.getLine();
+            }
+            int line = token.getType() == TokenConstants.FILE_END ? lines.size() : currentLine;
+            List<Token> tokens = lineNumbersToTokens.containsKey(line) ? lineNumbersToTokens.get(line) : new ArrayList<>();
+            tokens.add(token);
+            lineNumbersToTokens.put(line, tokens);
+        }
 
         // create LineData for each line -- 1-based line index
         Stream<Integer> lineNumbers = PRINT_EMPTY_LINES ? IntStream.range(1, lines.size() + 1).boxed() : lineNumbersToTokens.keySet().stream();
@@ -132,8 +144,8 @@ public final class TokenPrinter {
                 .toList();
     }
 
-    private static Map<String, List<Token>> groupTokensByFile(TokenList tokenList) {
-        return tokenList.allTokens().stream().collect(Collectors.groupingBy(Token::getFile));
+    private static Map<String, List<Token>> groupTokensByFile(List<Token> tokens) {
+        return tokens.stream().collect(Collectors.groupingBy(Token::getFile));
     }
 
     /**

jplag.frontend-utils/src/test/java/de/jplag/TokenPrinterTest.java

@@ -2,9 +2,12 @@ package de.jplag;
 
 import static de.jplag.TokenConstants.FILE_END;
 import static de.jplag.simple.TestTokenConstants.STRING;
-import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
@@ -25,55 +28,55 @@ class TokenPrinterTest {
     void printMockDirectoriesAsSubmissions() {
 
         // See TokenPrinterTest.txt for the intended behaviour
-        TokenList tokens = new TokenList();
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 1, 1, "STRING".length()));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 2, 1, "STRING".length() + 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 3, 1, "STRING".length() + 2));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 4, 1, "STRING".length() + 10));
+        List<Token> tokens = new ArrayList<>();
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 1, 1, "STRING".length()));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 2, 1, "STRING".length() + 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 3, 1, "STRING".length() + 2));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 4, 1, "STRING".length() + 10));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 6, 3, 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 7, 9, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 6, 3, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 7, 9, 1));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 9, 1, 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 9, 10, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 9, 1, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 9, 10, 1));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 10, 1, 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 10, 5, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 10, 1, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 10, 5, 1));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 12, 1, 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 10, 5, 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 12, 10, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 12, 1, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 12, 5, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 12, 10, 1));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 14, 10, 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 14, 5, 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 14, 1, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 14, 10, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 14, 5, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 14, 1, 1));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 16, -5, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 16, -5, 1));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 19, 100, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 19, 100, 1));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 22, 1, 100));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 22, 1, 100));
 
-        tokens.addToken(new TestToken(FILE_END, TEST_FILE_NAME, 24, 1, -1));
+        tokens.add(new TestToken(FILE_END, TEST_FILE_NAME, Token.NO_VALUE, Token.NO_VALUE, Token.NO_VALUE));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 100, 1, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 100, 1, 1));
 
         String output = TokenPrinter.printTokens(tokens, TEST_FILE_LOCATION.toFile());
-        logger.debug(output); // no additional newline required
+        logger.info(output); // no additional newline required
 
         testOutputCorrectness(TEST_FILE_NAME, tokens, output);
     }
 
-    private static void testOutputCorrectness(String fileName, TokenList tokens, String output) {
-        int lineIndex = 0;
+    private static void testOutputCorrectness(String fileName, List<Token> tokens, String output) {
+        int lineIndex = -1;
         int tokenIndex = 0;
-        boolean seenFileName = false;
         for (String line : output.lines().toList()) {
             if (line.isEmpty()) {
                 continue;
-            } else if (lineIndex == 0 && line.equals(fileName)) {
-                seenFileName = true;
-            } else if (line.startsWith("" + (lineIndex + 1))) {
+            } else if (lineIndex == -1) {
+                assertEquals(fileName, line);
+                lineIndex = 0;
+            } else if (line.startsWith(String.valueOf(lineIndex + 1))) {
                 lineIndex++;
             } else {
                 line = line.trim();
@@ -82,17 +85,15 @@ class TokenPrinterTest {
                     if (lineToken.isEmpty()) {
                         continue;
                     }
-                    Token currentToken = tokens.getToken(tokenIndex);
-                    if (lineToken.equalsIgnoreCase(currentToken.toString()) && currentToken.getLine() == lineIndex) {
-                        tokenIndex++;
-                    } else {
-                        fail("Expected token %s, but found %s".formatted(currentToken, lineToken));
+                    Token currentToken = tokens.get(tokenIndex);
+                    assertTrue(lineToken.equalsIgnoreCase(currentToken.toString()), "expected: %s, actual: %s".formatted(lineToken, currentToken));
+                    if (currentToken.getLine() != Token.NO_VALUE) {
+                        assertEquals(lineIndex, currentToken.getLine(), "invalid line for token " + currentToken);
                     }
+                    tokenIndex++;
                 }
             }
         }
-        if (!seenFileName) {
-            fail("Expected file name");
-        }
+        assertEquals(tokens.size() - 1, tokenIndex, "incorrect number of tokens printed");
     }
 }

jplag.frontend.cpp/src/main/java/de/jplag/cpp/Language.java

@@ -1,10 +1,11 @@
 package de.jplag.cpp;
 
 import java.io.File;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 @MetaInfServices(de.jplag.Language.class)
 public class Language implements de.jplag.Language {
@@ -37,7 +38,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File dir, String[] files) {
+    public List<Token> parse(File dir, String[] files) {
         return this.scanner.scan(dir, files);
     }
 

jplag.frontend.cpp/src/main/java/de/jplag/cpp/Scanner.java

@@ -1,14 +1,16 @@
 package de.jplag.cpp;
 
 import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenList;
+import de.jplag.TokenConstants;
 
 public class Scanner extends AbstractParser {
     private String currentFile;
 
-    private TokenList tokens;
+    private List<de.jplag.Token> tokens;
 
     /**
      * Creates the parser.
@@ -17,8 +19,8 @@ public class Scanner extends AbstractParser {
         super();
     }
 
-    public TokenList scan(File directory, String[] files) {
-        tokens = new TokenList();
+    public List<de.jplag.Token> scan(File directory, String[] files) {
+        tokens = new ArrayList<>();
         errors = 0;
         for (String currentFile : files) {
             this.currentFile = currentFile;
@@ -26,13 +28,13 @@ public class Scanner extends AbstractParser {
             if (!CPPScanner.scanFile(directory, currentFile, this)) {
                 errors++;
             }
-            tokens.addToken(new CPPToken(CPPTokenConstants.FILE_END, currentFile));
+            tokens.add(new CPPToken(TokenConstants.FILE_END, currentFile));
         }
         return tokens;
     }
 
     public void add(int type, Token token) {
         int length = token.endColumn - token.beginColumn + 1;
-        tokens.addToken(new CPPToken(type, currentFile, token.beginLine, token.beginColumn, length));
+        tokens.add(new CPPToken(type, currentFile, token.beginLine, token.beginColumn, length));
     }
 }

jplag.frontend.csharp-6/src/main/java/de/jplag/csharp/CSharpParserAdapter.java

@@ -3,6 +3,7 @@ package de.jplag.csharp;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.antlr.v4.runtime.CharStreams;
@@ -12,7 +13,8 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenList;
+import de.jplag.Token;
+import de.jplag.TokenConstants;
 import de.jplag.csharp.grammar.CSharpLexer;
 import de.jplag.csharp.grammar.CSharpParser;
 
@@ -22,7 +24,7 @@ import de.jplag.csharp.grammar.CSharpParser;
  * @author Timur Saglam
  */
 public class CSharpParserAdapter extends AbstractParser {
-    private TokenList tokens;
+    private List<Token> tokens;
     private String currentFile;
 
     /**
@@ -38,14 +40,14 @@ public class CSharpParserAdapter extends AbstractParser {
      * @param fileNames is the list of file names.
      * @return the list of parsed tokens.
      */
-    public TokenList parse(File directory, List<String> fileNames) {
-        tokens = new TokenList();
+    public List<Token> parse(File directory, List<String> fileNames) {
+        tokens = new ArrayList<>();
         errors = 0;
         for (String fileName : fileNames) {
             if (!parseFile(directory, fileName)) {
                 errors++;
             }
-            tokens.addToken(new CSharpToken(CSharpTokenConstants.FILE_END, fileName, -1, -1, -1));
+            tokens.add(new CSharpToken(TokenConstants.FILE_END, fileName, -1, -1, -1));
         }
         return tokens;
     }
@@ -77,6 +79,6 @@ public class CSharpParserAdapter extends AbstractParser {
     }
 
     /* package-private */ void addToken(int type, int line, int column, int length) {
-        tokens.addToken(new CSharpToken(type, currentFile, line, column, length));
+        tokens.add(new CSharpToken(type, currentFile, line, column, length));
     }
 }

jplag.frontend.csharp-6/src/main/java/de/jplag/csharp/Language.java

@@ -2,10 +2,11 @@ package de.jplag.csharp;
 
 import java.io.File;
 import java.util.Arrays;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 /**
  * C# language with full support of C# 6 features and below.
@@ -45,7 +46,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File dir, String[] files) {
+    public List<Token> parse(File dir, String[] files) {
         return parser.parse(dir, Arrays.asList(files));
     }
 

jplag.frontend.csharp-6/src/test/java/de/jplag/csharp/MinimalCSharpFrontendTest.java

@@ -16,7 +16,6 @@ import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.TokenPrinter;
 
 class MinimalCSharpFrontendTest {
@@ -44,12 +43,12 @@ class MinimalCSharpFrontendTest {
 
         // Parse test input
         String[] input = new String[] {TEST_SUBJECT};
-        TokenList result = frontend.parse(baseDirectory, input);
+        List<Token> result = frontend.parse(baseDirectory, input);
         logger.info(TokenPrinter.printTokens(result, baseDirectory));
 
         // Compare parsed tokens:
         assertEquals(expectedToken.size(), result.size());
-        List<Integer> actualToken = result.allTokens().stream().map(Token::getType).collect(toList());
+        List<Integer> actualToken = result.stream().map(Token::getType).collect(toList());
         assertEquals(expectedToken, actualToken);
     }
 

jplag.frontend.emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/parser/DynamicEcoreParser.java

@@ -23,7 +23,7 @@ public class DynamicEcoreParser extends EcoreParser {
     @Override
     public void addToken(int type, EObject source) {
         MetamodelToken token = new DynamicMetamodelToken(type, currentFile, source);
-        treeView.addToken(token, visitor.getCurrentTreeDepth(), NO_PREFIX);
-        tokens.addToken(token);
+        MetamodelToken metadataEnrichedToken = treeView.convertToMetadataEnrichedTokenAndAdd(token, visitor.getCurrentTreeDepth(), NO_PREFIX);
+        tokens.add(metadataEnrichedToken);
     }
 }

jplag.frontend.emf-metamodel-dynamic/src/test/java/de/jplag/emf/dynamic/MinimalDynamicMetamodelTest.java

@@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.File;
 import java.nio.file.Path;
+import java.util.List;
 import java.util.Optional;
 
 import org.junit.jupiter.api.AfterEach;
@@ -14,7 +15,7 @@ import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 import de.jplag.TokenPrinter;
 import de.jplag.testutils.FileUtil;
 import de.jplag.testutils.TokenUtils;
@@ -37,10 +38,10 @@ class MinimalDynamicMetamodelTest {
 
     @Test
     void testBookstoreMetamodels() {
-        TokenList result = frontend.parse(baseDirectory, TEST_SUBJECTS);
+        List<Token> result = frontend.parse(baseDirectory, TEST_SUBJECTS);
         logger.debug(TokenPrinter.printTokens(result, baseDirectory, Optional.of(Language.VIEW_FILE_SUFFIX)));
         logger.info(("Dynamic token set: " + DynamicMetamodelTokenConstants.getTokenStrings()));
-        logger.info("parsed tokens: " + result.allTokens().toString());
+        logger.info("parsed tokens: " + result.toString());
         assertEquals(7, DynamicMetamodelTokenConstants.getTokenStrings().size());
         assertEquals(64, result.size());
 

jplag.frontend.emf-metamodel/src/main/java/de/jplag/emf/Language.java

@@ -2,11 +2,12 @@ package de.jplag.emf;
 
 import java.io.File;
 import java.util.Arrays;
+import java.util.List;
 
 import org.eclipse.emf.ecore.EcorePackage;
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 import de.jplag.emf.parser.EcoreParser;
 
 /**
@@ -53,7 +54,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File dir, String[] files) {
+    public List<Token> parse(File dir, String[] files) {
         return parser.parse(dir, Arrays.asList(files));
     }
 

jplag.frontend.emf-metamodel/src/main/java/de/jplag/emf/MetamodelToken.java

@@ -36,6 +36,20 @@ public class MetamodelToken extends Token implements MetamodelTokenConstants {
         this.eObject = Optional.empty();
     }
 
+    /**
+     * Creates a token with column and length information.
+     * @param type is the token type.
+     * @param file is the name of the source code file.
+     * @param line is the line index in the source code where the token resides. Cannot be smaller than 1.
+     * @param column is the column index, meaning where the token starts in the line.
+     * @param length is the length of the token in the source code.
+     * @param eObject is the corresponding eObject in the model from which this token was extracted
+     */
+    public MetamodelToken(int type, String file, int line, int column, int length, Optional<EObject> eObject) {
+        super(type, file, line, column, length);
+        this.eObject = eObject;
+    }
+
     /**
      * @return the optional corresponding EObject of the token.
      */

jplag.frontend.emf-metamodel/src/main/java/de/jplag/emf/parser/EcoreParser.java

@@ -1,13 +1,14 @@
 package de.jplag.emf.parser;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.eclipse.emf.ecore.EObject;
 
 import de.jplag.AbstractParser;
+import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.emf.Language;
 import de.jplag.emf.MetamodelToken;
 import de.jplag.emf.util.AbstractMetamodelVisitor;
@@ -19,7 +20,7 @@ import de.jplag.emf.util.MetamodelTreeView;
  * @author Timur Saglam
  */
 public class EcoreParser extends AbstractParser {
-    protected TokenList tokens;
+    protected List<Token> tokens;
     protected String currentFile;
     protected MetamodelTreeView treeView;
     protected AbstractMetamodelVisitor visitor;
@@ -37,9 +38,9 @@ public class EcoreParser extends AbstractParser {
      * @param fileNames is the list of file names.
      * @return the list of parsed tokens.
      */
-    public TokenList parse(File directory, List<String> fileNames) {
+    public List<Token> parse(File directory, List<String> fileNames) {
         errors = 0;
-        tokens = new TokenList();
+        tokens = new ArrayList<>();
         for (String fileName : fileNames) {
             currentFile = fileName;
             String filePath = fileName.isEmpty() ? directory.toString() : directory.toString() + File.separator + fileName;
@@ -62,7 +63,7 @@ public class EcoreParser extends AbstractParser {
                 visitor = createMetamodelVisitor();
                 visitor.visit(root);
             }
-            tokens.addToken(new MetamodelToken(TokenConstants.FILE_END, currentFile));
+            tokens.add(new MetamodelToken(TokenConstants.FILE_END, currentFile));
             treeView.writeToFile(Language.VIEW_FILE_SUFFIX);
         }
     }
@@ -77,8 +78,8 @@ public class EcoreParser extends AbstractParser {
 
     public void addToken(int type, EObject source, String prefix) {
         MetamodelToken token = new MetamodelToken(type, currentFile, source);
-        treeView.addToken(token, visitor.getCurrentTreeDepth(), prefix);
-        tokens.addToken(token);
+        MetamodelToken metadataEnrichedToken = treeView.convertToMetadataEnrichedTokenAndAdd(token, visitor.getCurrentTreeDepth(), prefix);
+        tokens.add(metadataEnrichedToken);
     }
 
     public void addToken(int type, EObject source) {

jplag.frontend.emf-metamodel/src/main/java/de/jplag/emf/util/MetamodelTreeView.java

@@ -4,11 +4,14 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.util.Optional;
 
 import org.eclipse.emf.ecore.ENamedElement;
+import org.eclipse.emf.ecore.EObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import de.jplag.Token;
 import de.jplag.emf.MetamodelToken;
 
 /**
@@ -34,12 +37,18 @@ public class MetamodelTreeView {
     }
 
     /**
-     * Adds a token to the view, thus adding the index information to the token.
+     * Adds a token to the view, thus adding the index information to the token. Returns a new token enriched with the index
+     * metadata.
      * @param token is the token to add.
      * @param treeDepth is the current containment tree depth, required for the indentation.
      */
-    public void addToken(MetamodelToken token, int treeDepth, String prefix) {
-        token.getEObject().ifPresent(it -> {
+    public MetamodelToken convertToMetadataEnrichedTokenAndAdd(MetamodelToken token, int treeDepth, String prefix) {
+        int length = Token.NO_VALUE;
+        int line = Token.NO_VALUE;
+        int column = Token.NO_VALUE;
+        Optional<EObject> optionalEObject = token.getEObject();
+        if (optionalEObject.isPresent()) {
+            EObject eObject = optionalEObject.get();
             if (prefix.isEmpty() && treeDepth > 0) {
                 lineIndex++;
                 columnIndex = 0;
@@ -47,10 +56,10 @@ public class MetamodelTreeView {
             }
 
             String tokenText = token.toString();
-            if (it instanceof ENamedElement element) {
+            if (eObject instanceof ENamedElement element) {
                 tokenText = element.getName() + " : " + tokenText;
             }
-            token.setLength(tokenText.length());
+            length = tokenText.length();
 
             if (prefix.isEmpty()) {
                 for (int i = 0; i < treeDepth; i++) {
@@ -63,12 +72,12 @@ public class MetamodelTreeView {
                 columnIndex += prefix.length();
             }
 
-            token.setLine(lineIndex + 1);
-            token.setColumn(columnIndex + 1);
+            line = lineIndex + 1;
+            column = columnIndex + 1;
 
             columnIndex += tokenText.length();
-
-        });
+        }
+        return new MetamodelToken(token.getType(), token.getFile(), line, column, length, token.getEObject());
     }
 
     /**

jplag.frontend.emf-metamodel/src/test/java/de/jplag/emf/MinimalMetamodelTest.java

@@ -6,6 +6,7 @@ import java.io.File;
 import java.lang.reflect.Field;
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.List;
 import java.util.Optional;
 
 import org.junit.jupiter.api.AfterEach;
@@ -14,7 +15,7 @@ import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 import de.jplag.TokenPrinter;
 import de.jplag.testutils.FileUtil;
 import de.jplag.testutils.TokenUtils;
@@ -37,13 +38,13 @@ class MinimalMetamodelTest {
 
     @Test
     void testBookstoreMetamodels() {
-        TokenList result = frontend.parse(baseDirectory, TEST_SUBJECTS);
+        List<Token> result = frontend.parse(baseDirectory, TEST_SUBJECTS);
 
         logger.debug(TokenPrinter.printTokens(result, baseDirectory, Optional.of(Language.VIEW_FILE_SUFFIX)));
         Field[] fields = MetamodelTokenConstants.class.getFields();
         var constants = Arrays.stream(fields).map(Field::getName).filter(it -> !it.equals("NUM_DIFF_TOKENS")).toList();
         logger.info(("Handcrafted token set: " + constants));
-        logger.info("Parsed tokens: " + result.allTokens().toString());
+        logger.info("Parsed tokens: " + result.toString());
         assertEquals(21, constants.size());
         assertEquals(43, result.size());
 

jplag.frontend.golang/src/main/java/de/jplag/golang/GoParserAdapter.java

@@ -3,6 +3,8 @@ package de.jplag.golang;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
@@ -11,16 +13,16 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenList;
+import de.jplag.Token;
 import de.jplag.golang.grammar.GoLexer;
 import de.jplag.golang.grammar.GoParser;
 
 public class GoParserAdapter extends AbstractParser {
     private String currentFile;
-    private TokenList tokens;
+    private List<Token> tokens;
 
-    public TokenList parse(File directory, String[] fileNames) {
-        tokens = new TokenList();
+    public List<Token> parse(File directory, String[] fileNames) {
+        tokens = new ArrayList<>();
         for (String file : fileNames) {
             if (!parseFile(directory, file)) {
                 errors++;
@@ -54,6 +56,6 @@ public class GoParserAdapter extends AbstractParser {
     }
 
     public void addToken(int tokenType, int line, int column, int length) {
-        tokens.addToken(new GoToken(tokenType, currentFile, line, column, length));
+        tokens.add(new GoToken(tokenType, currentFile, line, column, length));
     }
 }

jplag.frontend.golang/src/main/java/de/jplag/golang/Language.java

@@ -1,10 +1,11 @@
 package de.jplag.golang;
 
 import java.io.File;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 @MetaInfServices(de.jplag.Language.class)
 public class Language implements de.jplag.Language {
@@ -40,7 +41,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File directory, String[] files) {
+    public List<Token> parse(File directory, String[] files) {
         return parserAdapter.parse(directory, files);
     }
 

jplag.frontend.golang/src/test/java/de/jplag/golang/GoFrontendTest.java

@@ -21,7 +21,6 @@ import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.TokenPrinter;
 
 class GoFrontendTest {
@@ -58,7 +57,7 @@ class GoFrontendTest {
     @Test
     void parseTestFiles() {
         for (String fileName : testFiles) {
-            TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
+            List<Token> tokens = language.parse(testFileLocation, new String[] {fileName});
             String output = TokenPrinter.printTokens(tokens, testFileLocation);
             logger.info(output);
 
@@ -87,9 +86,9 @@ class GoFrontendTest {
     /**
      * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
      * @param fileName a code sample file name
-     * @param tokens the TokenList generated from the sample
+     * @param tokens the list of tokens generated from the sample
      */
-    private void testSourceCoverage(String fileName, TokenList tokens) {
+    private void testSourceCoverage(String fileName, List<Token> tokens) {
         File testFile = new File(testFileLocation, fileName);
 
         List<String> lines = null;
@@ -103,7 +102,7 @@ class GoFrontendTest {
         // All lines that contain code
         var codeLines = getCodeLines(lines);
         // All lines that contain a token
-        var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().boxed().toList();
+        var tokenLines = tokens.stream().map(Token::getLine).distinct().toList();
 
         if (codeLines.size() > tokenLines.size()) {
             List<Integer> missedLinesIndices = new ArrayList<>(codeLines);
@@ -153,11 +152,11 @@ class GoFrontendTest {
 
     /**
      * Confirms that all Token types are 'reachable' with a complete code example.
-     * @param tokens TokenList which is supposed to contain all types of tokens
+     * @param tokens list of tokens which is supposed to contain all types of tokens
      * @param fileName The file name of the complete code example
      */
-    private void testTokenCoverage(TokenList tokens, String fileName) {
-        var foundTokens = tokens.allTokens().stream().parallel().mapToInt(Token::getType).sorted().distinct().boxed().toList();
+    private void testTokenCoverage(List<Token> tokens, String fileName) {
+        var foundTokens = tokens.stream().parallel().map(Token::getType).sorted().distinct().toList();
 
         // Exclude SEPARATOR_TOKEN, as it does not occur
         var missingTokenTypes = IntStream.range(0, GoTokenConstants.NUM_DIFF_TOKENS).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).boxed()

jplag.frontend.java/src/main/java/de/jplag/java/Language.java

@@ -1,10 +1,11 @@
 package de.jplag.java;
 
 import java.io.File;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 /**
  * Language for Java 9 and newer.
@@ -40,7 +41,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File directory, String[] files) {
+    public List<Token> parse(File directory, String[] files) {
         return this.parser.parse(directory, files);
     }
 

jplag.frontend.java/src/main/java/de/jplag/java/Parser.java

@@ -1,13 +1,15 @@
 package de.jplag.java;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 public class Parser extends AbstractParser {
-    private TokenList tokens;
+    private List<Token> tokens;
 
     /**
      * Creates the parser.
@@ -16,8 +18,8 @@ public class Parser extends AbstractParser {
         super();
     }
 
-    public TokenList parse(File directory, String[] files) {
-        tokens = new TokenList();
+    public List<Token> parse(File directory, String[] files) {
+        tokens = new ArrayList<>();
         errors = 0;
         var pathedFiles = Arrays.stream(files).map(it -> new File(directory, it)).toList();
         errors += new JavacAdapter().parseFiles(directory, pathedFiles, this);
@@ -25,7 +27,7 @@ public class Parser extends AbstractParser {
     }
 
     public void add(int type, String filename, long line, long column, long length) {
-        tokens.addToken(new JavaToken(type, filename, (int) line, (int) column, (int) length));
+        tokens.add(new JavaToken(type, filename, (int) line, (int) column, (int) length));
     }
 
     public void increaseErrors() {

jplag.frontend.kotlin/src/main/java/de/jplag/kotlin/KotlinParserAdapter.java

@@ -3,6 +3,8 @@ package de.jplag.kotlin;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
@@ -11,8 +13,8 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
+import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.kotlin.grammar.KotlinLexer;
 import de.jplag.kotlin.grammar.KotlinParser;
 
@@ -20,7 +22,7 @@ public class KotlinParserAdapter extends AbstractParser {
 
     public static final int NOT_SET = -1;
     private String currentFile;
-    private TokenList tokens;
+    private List<Token> tokens;
 
     /**
      * Creates the KotlinParserAdapter
@@ -30,18 +32,18 @@ public class KotlinParserAdapter extends AbstractParser {
     }
 
     /**
-     * Parsers a list of files into a single {@link TokenList}.
+     * Parsers a list of files into a single list of {@link Token}s.
      * @param directory the directory of the files.
      * @param fileNames the file names of the files.
-     * @return a {@link TokenList} containing all tokens of all files.
+     * @return a list containing all tokens of all files.
      */
-    public TokenList parse(File directory, String[] fileNames) {
-        tokens = new TokenList();
+    public List<Token> parse(File directory, String[] fileNames) {
+        tokens = new ArrayList<>();
         for (String file : fileNames) {
             if (!parseFile(directory, file)) {
                 errors++;
             }
-            tokens.addToken(new KotlinToken(TokenConstants.FILE_END, file, NOT_SET, NOT_SET, NOT_SET));
+            tokens.add(new KotlinToken(TokenConstants.FILE_END, file, NOT_SET, NOT_SET, NOT_SET));
         }
         return tokens;
     }
@@ -71,13 +73,13 @@ public class KotlinParserAdapter extends AbstractParser {
     }
 
     /**
-     * Adds a new {@link de.jplag.Token} to the current {@link TokenList}.
-     * @param tokenType the type of the new {@link de.jplag.Token}
+     * Adds a new {@link Token} to the current token list.
+     * @param tokenType the type of the new {@link Token}
      * @param line the line of the Token in the current file
      * @param column the start column of the Token in the line
      * @param length the length of the Token
      */
     /* package-private */ void addToken(int tokenType, int line, int column, int length) {
-        tokens.addToken(new KotlinToken(tokenType, currentFile, line, column, length));
+        tokens.add(new KotlinToken(tokenType, currentFile, line, column, length));
     }
 }

jplag.frontend.kotlin/src/main/java/de/jplag/kotlin/Language.java

@@ -1,10 +1,11 @@
 package de.jplag.kotlin;
 
 import java.io.File;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 /**
  * This represents the Kotlin language as a language supported by JPlag.
@@ -43,7 +44,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File directory, String[] files) {
+    public List<Token> parse(File directory, String[] files) {
         return parserAdapter.parse(directory, files);
     }
 

jplag.frontend.kotlin/src/test/java/de/jplag/kotlin/KotlinFrontendTest.java

@@ -18,7 +18,6 @@ import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.TokenPrinter;
 
 class KotlinFrontendTest {
@@ -58,7 +57,7 @@ class KotlinFrontendTest {
     @Test
     void parseTestFiles() {
         for (String fileName : testFiles) {
-            TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
+            List<Token> tokens = language.parse(testFileLocation, new String[] {fileName});
             String output = TokenPrinter.printTokens(tokens, testFileLocation);
             logger.info(output);
 
@@ -89,9 +88,9 @@ class KotlinFrontendTest {
     /**
      * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
      * @param fileName a code sample file name
-     * @param tokens the TokenList generated from the sample
+     * @param tokens the list of tokens generated from the sample
      */
-    private void testSourceCoverage(String fileName, TokenList tokens) {
+    private void testSourceCoverage(String fileName, List<Token> tokens) {
         File testFile = new File(testFileLocation, fileName);
 
         try {
@@ -100,7 +99,7 @@ class KotlinFrontendTest {
             // All lines that contain code
             var codeLines = getCodeLines(lines);
             // All lines that contain token
-            var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().toArray();
+            var tokenLines = tokens.stream().mapToInt(Token::getLine).filter(line -> line != Token.NO_VALUE).distinct().toArray();
 
             if (codeLines.length > tokenLines.length) {
                 var diffLine = IntStream.range(0, codeLines.length)
@@ -149,11 +148,11 @@ class KotlinFrontendTest {
 
     /**
      * Confirms that all Token types are 'reachable' with a complete code example.
-     * @param tokens TokenList which is supposed to contain all types of tokens
+     * @param tokens list of tokens which is supposed to contain all types of tokens
      * @param fileName The file name of the complete code example
      */
-    private void testTokenCoverage(TokenList tokens, String fileName) {
-        var foundTokens = tokens.allTokens().stream().parallel().mapToInt(Token::getType).sorted().distinct().toArray();
+    private void testTokenCoverage(List<Token> tokens, String fileName) {
+        var foundTokens = tokens.stream().parallel().mapToInt(Token::getType).sorted().distinct().toArray();
         // Exclude SEPARATOR_TOKEN, as it does not occur
         var allTokens = IntStream.range(0, KotlinTokenConstants.NUMBER_DIFF_TOKENS).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).toArray();
 

jplag.frontend.python-3/src/main/java/de/jplag/python3/Language.java

@@ -1,10 +1,11 @@
 package de.jplag.python3;
 
 import java.io.File;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 @MetaInfServices(de.jplag.Language.class)
 public class Language implements de.jplag.Language {
@@ -38,7 +39,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File dir, String[] files) {
+    public List<Token> parse(File dir, String[] files) {
         return this.parser.parse(dir, files);
     }
 

jplag.frontend.python-3/src/main/java/de/jplag/python3/Parser.java

@@ -4,6 +4,8 @@ import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CharStreams;
@@ -13,14 +15,14 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenList;
+import de.jplag.TokenConstants;
 import de.jplag.python3.grammar.Python3Lexer;
 import de.jplag.python3.grammar.Python3Parser;
 import de.jplag.python3.grammar.Python3Parser.File_inputContext;
 
 public class Parser extends AbstractParser {
 
-    private TokenList tokens = new TokenList();
+    private List<de.jplag.Token> tokens;
     private String currentFile;
 
     /**
@@ -30,15 +32,15 @@ public class Parser extends AbstractParser {
         super();
     }
 
-    public TokenList parse(File directory, String[] files) {
-        tokens = new TokenList();
+    public List<de.jplag.Token> parse(File directory, String[] files) {
+        tokens = new ArrayList<>();
         errors = 0;
         for (String file : files) {
             logger.trace("Parsing file {}", file);
             if (!parseFile(directory, file)) {
                 errors++;
             }
-            tokens.addToken(new Python3Token(Python3TokenConstants.FILE_END, file, -1, -1, -1));
+            tokens.add(new Python3Token(TokenConstants.FILE_END, file, -1, -1, -1));
         }
         return tokens;
     }
@@ -77,12 +79,12 @@ public class Parser extends AbstractParser {
     }
 
     public void add(int type, Token token) {
-        tokens.addToken(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(), token.getCharPositionInLine() + 1,
+        tokens.add(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(), token.getCharPositionInLine() + 1,
                 token.getText().length()));
     }
 
     public void addEnd(int type, Token token) {
-        tokens.addToken(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(),
-                tokens.getToken(tokens.size() - 1).getColumn() + 1, 0));
+        tokens.add(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(),
+                tokens.get(tokens.size() - 1).getColumn() + 1, 0));
     }
 }

jplag.frontend.rlang/src/main/java/de/jplag/rlang/Language.java

@@ -1,10 +1,11 @@
 package de.jplag.rlang;
 
 import java.io.File;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 /**
  * This represents the R language as a language supported by JPlag.
@@ -43,7 +44,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File directory, String[] files) {
+    public List<Token> parse(File directory, String[] files) {
         return parserAdapter.parse(directory, files);
     }
 

jplag.frontend.rlang/src/main/java/de/jplag/rlang/RParserAdapter.java

@@ -3,6 +3,8 @@ package de.jplag.rlang;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
@@ -11,7 +13,7 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenList;
+import de.jplag.Token;
 import de.jplag.rlang.grammar.RFilter;
 import de.jplag.rlang.grammar.RLexer;
 import de.jplag.rlang.grammar.RParser;
@@ -23,7 +25,7 @@ import de.jplag.rlang.grammar.RParser;
 public class RParserAdapter extends AbstractParser implements RTokenConstants {
 
     private String currentFile;
-    private TokenList tokens;
+    private List<Token> tokens;
 
     /**
      * Creates the RParserAdapter
@@ -33,19 +35,19 @@ public class RParserAdapter extends AbstractParser implements RTokenConstants {
     }
 
     /**
-     * Parsers a list of files into a single {@link TokenList}.
+     * Parsers a list of files into a single token list of {@link Token}s.
      * @param directory the directory of the files.
      * @param fileNames the file names of the files.
-     * @return a {@link TokenList} containing all tokens of all files.
+     * @return a list containing all tokens of all files.
      */
-    public TokenList parse(File directory, String[] fileNames) {
-        tokens = new TokenList();
+    public List<Token> parse(File directory, String[] fileNames) {
+        tokens = new ArrayList<>();
         errors = 0;
         for (String fileName : fileNames) {
             if (!parseFile(directory, fileName)) {
                 errors++;
             }
-            tokens.addToken(new RToken(FILE_END, fileName, -1, -1, -1));
+            tokens.add(new RToken(FILE_END, fileName, -1, -1, -1));
         }
         return tokens;
     }
@@ -82,14 +84,14 @@ public class RParserAdapter extends AbstractParser implements RTokenConstants {
     }
 
     /**
-     * Adds a new {@link de.jplag.Token} to the current {@link TokenList}.
-     * @param type the type of the new {@link de.jplag.Token}
+     * Adds a new {@link Token} to the current token list.
+     * @param type the type of the new {@link Token}
      * @param line the line of the Token in the current file
      * @param start the start column of the Token in the line
      * @param length the length of the Token
      */
     /* package-private */ void addToken(int type, int line, int start, int length) {
-        tokens.addToken(new RToken(type, currentFile, line, start, length));
+        tokens.add(new RToken(type, currentFile, line, start, length));
 
     }
 }

jplag.frontend.rlang/src/test/java/de/jplag/rlang/RFrontendTest.java

@@ -17,7 +17,6 @@ import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.TokenPrinter;
 
 class RFrontendTest {
@@ -46,7 +45,7 @@ class RFrontendTest {
     @Test
     void parseTestFiles() {
         for (String fileName : testFiles) {
-            TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
+            List<Token> tokens = language.parse(testFileLocation, new String[] {fileName});
             String output = TokenPrinter.printTokens(tokens, testFileLocation);
             logger.info(output);
 
@@ -59,9 +58,9 @@ class RFrontendTest {
     /**
      * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
      * @param fileName a code sample file name
-     * @param tokens the TokenList generated from the sample
+     * @param tokens the list of tokens generated from the sample
      */
-    private void testSourceCoverage(String fileName, TokenList tokens) {
+    private void testSourceCoverage(String fileName, List<Token> tokens) {
         File testFile = new File(testFileLocation, fileName);
 
         try {
@@ -71,7 +70,7 @@ class RFrontendTest {
             // All lines that contain code
             var codeLines = IntStream.range(1, lines.size() + 1).filter(idx -> !lines.get(idx - 1).matches(emptyLineExpression)).toArray();
             // All lines that contain token
-            var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().toArray();
+            var tokenLines = tokens.stream().mapToInt(Token::getLine).filter(line -> line != Token.NO_VALUE).distinct().toArray();
 
             if (codeLines.length > tokenLines.length) {
                 var diffLine = IntStream.range(0, codeLines.length)
@@ -88,11 +87,11 @@ class RFrontendTest {
 
     /**
      * Confirms that all Token types are 'reachable' with a complete code example.
-     * @param tokens TokenList which is supposed to contain all types of tokens
+     * @param tokens list of tokens which is supposed to contain all types of tokens
      * @param fileName The file name of the complete code example
      */
-    private void testTokenCoverage(TokenList tokens, String fileName) {
-        var foundTokens = tokens.allTokens().stream().parallel().mapToInt(Token::getType).sorted().distinct().toArray();
+    private void testTokenCoverage(List<Token> tokens, String fileName) {
+        var foundTokens = tokens.stream().parallel().mapToInt(Token::getType).sorted().distinct().toArray();
         // Exclude SEPARATOR_TOKEN, as it does not occur
         var allTokens = IntStream.range(0, RTokenConstants.NUM_DIFF_TOKENS).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).toArray();
 

jplag.frontend.rust/src/main/java/de/jplag/rust/Language.java

@@ -1,8 +1,9 @@
 package de.jplag.rust;
 
 import java.io.File;
+import java.util.List;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 public class Language implements de.jplag.Language {
 
@@ -38,7 +39,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File directory, String[] files) {
+    public List<Token> parse(File directory, String[] files) {
         return parserAdapter.parse(directory, files);
     }
 

jplag.frontend.rust/src/main/java/de/jplag/rust/RustParserAdapter.java

@@ -3,6 +3,8 @@ package de.jplag.rust;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
@@ -11,8 +13,8 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
+import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.rust.grammar.RustLexer;
 import de.jplag.rust.grammar.RustParser;
 
@@ -20,22 +22,22 @@ public class RustParserAdapter extends AbstractParser {
 
     private static final int NOT_SET = -1;
     private String currentFile;
-    private TokenList tokens;
+    private List<Token> tokens;
 
     /**
-     * Parsers a list of files into a single {@link TokenList}.
+     * Parsers a list of files into a single list of {@link Token}s.
      * @param directory the directory of the files.
      * @param fileNames the file names of the files.
-     * @return a {@link TokenList} containing all tokens of all files.
+     * @return a list containing all tokens of all files.
      */
-    public TokenList parse(File directory, String[] fileNames) {
-        tokens = new TokenList();
+    public List<Token> parse(File directory, String[] fileNames) {
+        tokens = new ArrayList<>();
         errors = 0;
         for (String fileName : fileNames) {
             if (!parseFile(directory, fileName)) {
                 errors++;
             }
-            tokens.addToken(new RustToken(TokenConstants.FILE_END, fileName, NOT_SET, NOT_SET, NOT_SET));
+            tokens.add(new RustToken(TokenConstants.FILE_END, fileName, NOT_SET, NOT_SET, NOT_SET));
         }
         return tokens;
     }
@@ -68,14 +70,14 @@ public class RustParserAdapter extends AbstractParser {
     }
 
     /**
-     * Adds a new {@link de.jplag.Token} to the current {@link TokenList}.
-     * @param type the type of the new {@link de.jplag.Token}
+     * Adds a new {@link Token} to the current token list.
+     * @param type the type of the new {@link Token}
      * @param line the line of the Token in the current file
      * @param start the start column of the Token in the line
      * @param length the length of the Token
      */
     /* package-private */ void addToken(int type, int line, int start, int length) {
-        tokens.addToken(new RustToken(type, currentFile, line, start, length));
+        tokens.add(new RustToken(type, currentFile, line, start, length));
 
     }
 }

jplag.frontend.rust/src/test/java/de/jplag/rust/RustFrontendTest.java

@@ -17,7 +17,6 @@ import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.TokenPrinter;
 
 class RustFrontendTest {
@@ -52,7 +51,7 @@ class RustFrontendTest {
     @Test
     void parseTestFiles() {
         for (String fileName : testFiles) {
-            TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
+            List<Token> tokens = language.parse(testFileLocation, new String[] {fileName});
             String output = TokenPrinter.printTokens(tokens, testFileLocation);
             logger.info(output);
 
@@ -65,9 +64,9 @@ class RustFrontendTest {
     /**
      * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
      * @param fileName a code sample file name
-     * @param tokens the TokenList generated from the sample
+     * @param tokens the list of tokens generated from the sample
      */
-    private void testSourceCoverage(String fileName, TokenList tokens) {
+    private void testSourceCoverage(String fileName, List<Token> tokens) {
         File testFile = new File(testFileLocation, fileName);
 
         try {
@@ -76,7 +75,7 @@ class RustFrontendTest {
             // All lines that contain code
             var codeLines = new ArrayList<>(getCodeLines(lines));
             // All lines that contain token
-            var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().boxed().toList();
+            var tokenLines = tokens.stream().map(Token::getLine).filter(line -> line != Token.NO_VALUE).distinct().toList();
 
             // Keep only lines that have no tokens
             codeLines.removeAll(tokenLines);
@@ -122,11 +121,11 @@ class RustFrontendTest {
 
     /**
      * Confirms that all Token types are 'reachable' with a complete code example.
-     * @param tokens TokenList which is supposed to contain all types of tokens
+     * @param tokens list of tokens which is supposed to contain all types of tokens
      * @param fileName The file name of the complete code example
      */
-    private void testTokenCoverage(TokenList tokens, String fileName) {
-        var foundTokens = tokens.allTokens().stream().mapToInt(Token::getType).distinct().boxed().toList();
+    private void testTokenCoverage(List<Token> tokens, String fileName) {
+        var foundTokens = tokens.stream().map(Token::getType).distinct().toList();
         var allTokens = IntStream.range(0, RustTokenConstants.NUMBER_DIFF_TOKENS).boxed().toList();
         allTokens = new ArrayList<>(allTokens);
 

jplag.frontend.scala/src/main/scala/de/jplag/scala/Language.scala

@@ -1,8 +1,9 @@
 package de.jplag.scala
 
-import de.jplag.TokenList
+import de.jplag.Token
 
 import java.io.File
+import scala.collection.JavaConverters._
 
 import org.kohsuke.MetaInfServices
 
@@ -18,7 +19,7 @@ class Language extends de.jplag.Language {
 
   override def minimumTokenMatch = 8
 
-  override def parse(dir: File, files: Array[String]): TokenList = this.parser.parse(dir, files)
+  override def parse(dir: File, files: Array[String]): java.util.List[Token] = this.parser.parse(dir, files).asJava
 
   override def hasErrors: Boolean = this.parser.hasErrors
 

jplag.frontend.scala/src/main/scala/de/jplag/scala/Parser.scala

@@ -1,16 +1,18 @@
 package de.jplag.scala
 
 import de.jplag.scala.ScalaTokenConstants._
-import de.jplag.{AbstractParser, TokenList}
+import de.jplag.{AbstractParser, Token}
 
 import java.io.File
+
+import scala.collection.mutable.ListBuffer 
 import scala.meta._
 
 
 class Parser extends AbstractParser {
     private var currentFile: String = _
 
-    private var tokens: TokenList = _
+    private var tokens: ListBuffer[Token] = _
 
     private val traverser: Traverser = new Traverser {
 
@@ -328,8 +330,8 @@ class Parser extends AbstractParser {
         }
     }
 
-    def parse(directory: File, files: Array[String]): TokenList = {
-        tokens = new TokenList
+    def parse(directory: File, files: Array[String]): List[Token] = {
+        tokens = ListBuffer()
         errors = 0
 
         for (file <- files) {
@@ -340,7 +342,7 @@ class Parser extends AbstractParser {
             System.gc()
         }
 
-        tokens
+        tokens.toList
     }
 
     private def parseFile(directory: File, fileName: String): Boolean = {
@@ -373,7 +375,7 @@ class Parser extends AbstractParser {
      * @param length    length of the occurrence in the file
      */
     private def add(tokenType: ScalaTokenConstants.Value, line: Int, column: Int, length: Int): Unit = {
-        tokens.addToken(new ScalaToken(tokenType.id, currentFile, line, column, length))
+        tokens += new ScalaToken(tokenType.id, currentFile, line, column, length)
     }
 
 
@@ -387,7 +389,7 @@ class Parser extends AbstractParser {
     private def add(tokenType: ScalaTokenConstants.Value, node: Tree, fromEnd: Boolean): Unit = {
         if (node.pos.text.nonEmpty) {
             // SELF type tokens with no text content mess up the sequence
-            tokens.addToken(new ScalaToken(tokenType, currentFile, node.pos, fromEnd))
+            tokens += new ScalaToken(tokenType, currentFile, node.pos, fromEnd)
         }
     }
 

jplag.frontend.scala/src/test/java/de/jplag/scala/ScalaFrontendTest.java

@@ -18,7 +18,6 @@ import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.TokenPrinter;
 
 class ScalaFrontendTest {
@@ -59,7 +58,7 @@ class ScalaFrontendTest {
     @Test
     void parseTestFiles() {
         for (String fileName : testFiles) {
-            TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
+            List<Token> tokens = language.parse(testFileLocation, new String[] {fileName});
             String output = TokenPrinter.printTokens(tokens, testFileLocation);
             logger.info(output);
 
@@ -90,9 +89,9 @@ class ScalaFrontendTest {
     /**
      * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
      * @param fileName a code sample file name
-     * @param tokens the TokenList generated from the sample
+     * @param tokens the list of tokens generated from the sample
      */
-    private void testSourceCoverage(String fileName, TokenList tokens) {
+    private void testSourceCoverage(String fileName, List<Token> tokens) {
         var testFile = new File(testFileLocation, fileName);
 
         try {
@@ -101,7 +100,7 @@ class ScalaFrontendTest {
             // All lines that contain code
             var codeLines = new ArrayList<>(getCodeLines(lines));
             // All lines that contain token
-            var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().boxed().toList();
+            var tokenLines = tokens.stream().map(Token::getLine).filter(line -> line != Token.NO_VALUE).distinct().toList();
 
             // Keep only lines that have no tokens
             codeLines.removeAll(tokenLines);
@@ -157,11 +156,11 @@ class ScalaFrontendTest {
 
     /**
      * Confirms that all Token types are 'reachable' with a complete code example.
-     * @param tokens TokenList which is supposed to contain all types of tokens
+     * @param tokens list of tokens which is supposed to contain all types of tokens
      * @param fileName The file name of the complete code example
      */
-    private void testTokenCoverage(TokenList tokens, String fileName) {
-        var foundTokens = tokens.allTokens().stream().parallel().mapToInt(Token::getType).distinct().boxed().toList();
+    private void testTokenCoverage(List<Token> tokens, String fileName) {
+        var foundTokens = tokens.stream().parallel().map(Token::getType).distinct().toList();
         var allTokens = IntStream.range(0, ScalaTokenConstants.numberOfTokens()).boxed().toList();
         allTokens = new ArrayList<>(allTokens);
 

jplag.frontend.scheme/src/main/java/de/jplag/scheme/Language.java

@@ -1,10 +1,11 @@
 package de.jplag.scheme;
 
 import java.io.File;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 @MetaInfServices(de.jplag.Language.class)
 public class Language implements de.jplag.Language {
@@ -37,7 +38,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File dir, String[] files) {
+    public List<Token> parse(File dir, String[] files) {
         return this.parser.parse(dir, files);
     }
 

jplag.frontend.scheme/src/main/java/de/jplag/scheme/Parser.java

@@ -1,14 +1,16 @@
 package de.jplag.scheme;
 
 import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenList;
+import de.jplag.TokenConstants;
 
 public class Parser extends AbstractParser {
     private String currentFile;
 
-    private TokenList tokens;
+    private List<de.jplag.Token> tokens;
 
     /**
      * Creates the parser.
@@ -17,22 +19,22 @@ public class Parser extends AbstractParser {
         super();
     }
 
-    public TokenList parse(File directory, String[] files) {
-        tokens = new TokenList();
+    public List<de.jplag.Token> parse(File directory, String[] files) {
+        tokens = new ArrayList<>();
         errors = 0;
         for (String file : files) {
             currentFile = file;
             logger.trace("Parsing file {}", file);
             if (!SchemeParser.parseFile(directory, file, null, this))
                 errors++;
-            tokens.addToken(new SchemeToken(SchemeTokenConstants.FILE_END, currentFile));
+            tokens.add(new SchemeToken(TokenConstants.FILE_END, currentFile));
         }
         return tokens;
     }
 
     public void add(int type, Token token) {
         int length = token.endColumn - token.beginColumn + 1;
-        tokens.addToken(new SchemeToken(type, currentFile, token.beginLine, token.endLine, length));
+        tokens.add(new SchemeToken(type, currentFile, token.beginLine, token.endLine, length));
     }
 
 }

jplag.frontend.text/src/main/java/de/jplag/text/Language.java

@@ -1,10 +1,11 @@
 package de.jplag.text;
 
 import java.io.File;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 /**
  * Language class for parsing (natural language) text. This language module employs a primitive approach where
@@ -42,7 +43,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File dir, String[] files) {
+    public List<Token> parse(File dir, String[] files) {
         return parserAdapter.parse(dir, files);
     }
 

jplag.frontend.text/src/main/java/de/jplag/text/ParserAdapter.java

@@ -4,13 +4,15 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
 import de.jplag.AbstractParser;
+import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 
 import edu.stanford.nlp.ling.CoreLabel;
 import edu.stanford.nlp.pipeline.CoreDocument;
@@ -26,7 +28,7 @@ public class ParserAdapter extends AbstractParser {
     private final StanfordCoreNLP pipeline;
     private int tokenTypeIndex = 2; // 0 is FILE_END token, 1 is SEPARATOR_TOKEN, so start at 2.
 
-    private TokenList tokens;
+    private List<Token> tokens;
     private String currentFile;
     private int currentLine;
     /**
@@ -40,15 +42,15 @@ public class ParserAdapter extends AbstractParser {
         this.pipeline = new StanfordCoreNLP(properties);
     }
 
-    public TokenList parse(File directory, String[] files) {
-        tokens = new TokenList();
+    public List<Token> parse(File directory, String[] files) {
+        tokens = new ArrayList<>();
         errors = 0;
         for (String file : files) {
             logger.trace("Parsing file {}", file);
             if (!parseFile(directory, file)) {
                 errors++;
             }
-            tokens.addToken(new TextToken(TokenConstants.FILE_END, file));
+            tokens.add(new TextToken(TokenConstants.FILE_END, file));
         }
         return tokens;
     }
@@ -106,7 +108,7 @@ public class ParserAdapter extends AbstractParser {
         int type = getTokenType(text);
         int column = label.beginPosition() - currentLineBreakIndex;
         int length = label.endPosition() - label.beginPosition();
-        tokens.addToken(new TextToken(text, type, currentFile, currentLine, column, length));
+        tokens.add(new TextToken(text, type, currentFile, currentLine, column, length));
     }
 
     private String readFile(Path filePath) {

jplag.frontend.text/src/test/java/jplag/text/TextFrontendTest.java

@@ -8,6 +8,7 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.junit.jupiter.api.BeforeEach;
@@ -19,7 +20,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
-import de.jplag.TokenList;
 import de.jplag.TokenPrinter;
 import de.jplag.text.Language;
 
@@ -43,12 +43,12 @@ class TextFrontendTest {
     void testParsingJavaDoc() {
         // Parse test input
         String[] input = new String[] {TEST_SUBJECT};
-        TokenList result = frontend.parse(baseDirectory, input);
+        List<Token> result = frontend.parse(baseDirectory, input);
         logger.info(TokenPrinter.printTokens(result, baseDirectory));
 
         // Compare parsed tokens:
         Map<Integer, Token> tokenTypes = new HashMap<>();
-        result.allTokens().forEach(it -> tokenTypes.put(it.getType(), it));
+        result.forEach(it -> tokenTypes.put(it.getType(), it));
 
         assertEquals(283, result.size());
         assertEquals(158, tokenTypes.values().size());
@@ -59,7 +59,7 @@ class TextFrontendTest {
     void testLineBreakInputs(String input, @TempDir Path tempDir) throws IOException {
         Path file = tempDir.resolve("input.txt");
         Files.writeString(file, input);
-        TokenList result = frontend.parse(tempDir.toFile(), new String[] {"input.txt"});
+        List<Token> result = frontend.parse(tempDir.toFile(), new String[] {"input.txt"});
         assertEquals(1, result.size());
     }
 
@@ -68,8 +68,8 @@ class TextFrontendTest {
     void testTokenAfterLineBreak(String input, @TempDir Path tempDir) throws IOException {
         Path file = tempDir.resolve("input.txt");
         Files.writeString(file, input);
-        TokenList result = frontend.parse(tempDir.toFile(), new String[] {"input.txt"});
-        assertEquals(2, result.getToken(0).getLine());
+        List<Token> result = frontend.parse(tempDir.toFile(), new String[] {"input.txt"});
+        assertEquals(2, result.get(0).getLine());
     }
 
 }

jplag/src/main/java/de/jplag/GreedyStringTiling.java

@@ -4,100 +4,69 @@ import static de.jplag.TokenConstants.FILE_END;
 import static de.jplag.TokenConstants.SEPARATOR_TOKEN;
 
 import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import de.jplag.options.JPlagOptions;
 
 /**
  * This class implements the Greedy String Tiling algorithm as introduced by Michael Wise. However, it is very specific
- * to the classes {@link TokenList}, {@link Token}, and {@link Match}. While this class was reworked, it still contains
- * some quirks from the initial version.
+ * to the classes {@link Token}, and {@link Match}.
  * @see <a href=
  * "https://www.researchgate.net/publication/262763983_String_Similarity_via_Greedy_String_Tiling_and_Running_Karp-Rabin_Matching">
  * String Similarity via Greedy String Tiling and Running Karp−Rabin Matching </a>
  */
 public class GreedyStringTiling {
 
-    private final JPlagOptions options;
+    private final int minimumMatchLength;
+    private final Map<Submission, SubsequenceHashLookupTable> cachedHashLookupTables = new IdentityHashMap<>();
+    private final Map<Submission, Set<Token>> baseCodeMarkings = new IdentityHashMap<>();
 
     public GreedyStringTiling(JPlagOptions options) {
-        this.options = options;
+        this.minimumMatchLength = options.minimumTokenMatch();
     }
 
     /**
-     * Creating hashes in linear time. The hash-code will be written in every Token for the next &lt;hashLength&gt; token
-     * (includes the Token itself).
-     * @param tokenList contains the tokens.
-     * @param hashLength is the hash length (condition: 1 &lt; hashLength &lt; 26)
-     * @param makeTable determines if a simple hash table is created in the structure.
+     * Compares the given submission with the base code submission. Marks the identified base code sections in the
+     * submission such that further comparisons do not generate matches for these parts. Must be called before generating a
+     * comparison with a regular submission for the given submission.
+     * @param submission is the submission to generate base-code markings for.
+     * @param baseCodeSubmission is the base code submission.
+     * @return the comparison of the submission with the base code submission.
      */
-    public void createHashes(TokenList tokenList, int hashLength, boolean makeTable) {
-        // Here the upper boundary of the hash length is set.
-        // It is determined by the number of bits of the 'int' data type and the number of tokens.
-        if (hashLength < 1) {
-            hashLength = 1;
+    public final JPlagComparison generateBaseCodeMarking(Submission submission, Submission baseCodeSubmission) {
+        JPlagComparison comparison = compare(submission, baseCodeSubmission);
+
+        List<Token> submissionTokenList = submission.getTokenList();
+        Set<Token> baseCodeMarking = new HashSet<>();
+        for (Match match : comparison.matches()) {
+            int startIndex = comparison.firstSubmission() == submission ? match.startOfFirst() : match.startOfSecond();
+            baseCodeMarking.addAll(submissionTokenList.subList(startIndex, startIndex + match.length()));
         }
-        hashLength = (hashLength < 26 ? hashLength : 25);
+        baseCodeMarkings.put(submission, baseCodeMarking);
 
-        if (tokenList.size() < hashLength) {
-            return;
-        }
+        // Remove the lookup table for the current submission to trigger a regeneration as hashes will change due to the new
+        // baseCodeMarking.
+        // This is a performance optimization to not suggest subsequences with baseCode for the matching.
+        // Removing this optimization would not change the result as the baseCode matches are additionally checked by validating
+        // that no match has a marked token (which baseCode-containing tokens are).
+        cachedHashLookupTables.remove(submission);
 
-        int modulo = ((1 << 6) - 1);   // Modulo 64!
-
-        int loops = tokenList.size() - hashLength;
-        tokenList.tokenHashes = (makeTable ? new TokenHashMap(3 * loops) : null);
-        int hash = 0;
-        int hashedLength = 0;
-        for (int i = 0; i < hashLength; i++) {
-            hash = (2 * hash) + (tokenList.getToken(i).type & modulo);
-            hashedLength++;
-            if (tokenList.getToken(i).isMarked()) {
-                hashedLength = 0;
-            }
-        }
-        int factor = (hashLength != 1 ? (2 << (hashLength - 2)) : 1);
-
-        if (makeTable) {
-            for (int i = 0; i < loops; i++) {
-                if (hashedLength >= hashLength) {
-                    tokenList.getToken(i).setHash(hash);
-                    tokenList.tokenHashes.put(hash, i);   // add into hashtable
-                } else {
-                    tokenList.getToken(i).setHash(-1);
-                }
-                hash -= factor * (tokenList.getToken(i).type & modulo);
-                hash = (2 * hash) + (tokenList.getToken(i + hashLength).type & modulo);
-                if (tokenList.getToken(i + hashLength).isMarked()) {
-                    hashedLength = 0;
-                } else {
-                    hashedLength++;
-                }
-            }
-        } else {
-            for (int i = 0; i < loops; i++) {
-                tokenList.getToken(i).setHash((hashedLength >= hashLength) ? hash : -1);
-                hash -= factor * (tokenList.getToken(i).type & modulo);
-                hash = (2 * hash) + (tokenList.getToken(i + hashLength).type & modulo);
-                if (tokenList.getToken(i + hashLength).isMarked()) {
-                    hashedLength = 0;
-                } else {
-                    hashedLength++;
-                }
-            }
-        }
-        tokenList.hashLength = hashLength;
+        return comparison;
     }
 
+    /**
+     * Compares the two submissions and generates matches between them. To exclude base code from the result, call
+     * {@link #generateBaseCodeMarking} with each submission beforehand.
+     * @param firstSubmission is one of the two submissions.
+     * @param secondSubmission is the other of the two submissions.
+     * @return the comparison between the two submissions.
+     */
     public final JPlagComparison compare(Submission firstSubmission, Submission secondSubmission) {
-        return swapAndCompare(firstSubmission, secondSubmission, false);
-    }
-
-    public final JPlagComparison compareWithBaseCode(Submission firstSubmission, Submission secondSubmission) {
-        return swapAndCompare(firstSubmission, secondSubmission, true);
-    }
-
-    private JPlagComparison swapAndCompare(Submission firstSubmission, Submission secondSubmission, boolean isBaseCodeComparison) {
         Submission smallerSubmission;
         Submission largerSubmission;
         if (firstSubmission.getTokenList().size() > secondSubmission.getTokenList().size()) {
@@ -107,129 +76,126 @@ public class GreedyStringTiling {
             smallerSubmission = firstSubmission;
             largerSubmission = secondSubmission;
         }
-        // if hashtable exists in first but not in second structure: flip around!
-        if (largerSubmission.getTokenList().tokenHashes == null && smallerSubmission.getTokenList().tokenHashes != null) {
-            Submission swap = smallerSubmission;
-            smallerSubmission = largerSubmission;
-            largerSubmission = swap;
-        }
-        return compare(smallerSubmission, largerSubmission, isBaseCodeComparison);
+        return compareInternal(smallerSubmission, largerSubmission);
     }
 
     /**
      * Compares two submissions. FILE_END is used as pivot
-     * @param firstSubmission is the submission with the smaller sequence.
-     * @param secondSubmission is the submission with the larger sequence.
-     * @param isBaseCodeComparison specifies whether one of the submissions is the base code.
+     * @param leftSubmission is the submission with the smaller sequence.
+     * @param rightSubmission is the submission with the larger sequence.
      * @return the comparison results.
      */
-    private JPlagComparison compare(Submission firstSubmission, Submission secondSubmission, boolean isBaseCodeComparison) {
-        // first and second refer to the list of tokens of the first and second submission:
-        TokenList first = firstSubmission.getTokenList();
-        TokenList second = secondSubmission.getTokenList();
+    private JPlagComparison compareInternal(Submission leftSubmission, Submission rightSubmission) {
+        List<Token> leftTokens = leftSubmission.getTokenList();
+        List<Token> rightTokens = rightSubmission.getTokenList();
 
-        // Initialize:
-        JPlagComparison comparison = new JPlagComparison(firstSubmission, secondSubmission);
-        int minimumTokenMatch = options.minimumTokenMatch(); // minimal required token match
-
-        if (first.size() <= minimumTokenMatch || second.size() <= minimumTokenMatch) { // <= because of pivots!
-            return comparison;
+        // comparison uses <= because it is assumed that the last token is a pivot (FILE_END)
+        if (leftTokens.size() <= minimumMatchLength || rightTokens.size() <= minimumMatchLength) {
+            return new JPlagComparison(leftSubmission, rightSubmission, List.of());
         }
 
-        markTokens(first, isBaseCodeComparison);
-        markTokens(second, isBaseCodeComparison);
+        Set<Token> leftMarkedTokens = initiallyMarkedTokens(leftSubmission);
+        Set<Token> rightMarkedTokens = initiallyMarkedTokens(rightSubmission);
 
-        // create hashes:
-        if (first.hashLength != minimumTokenMatch) {
-            createHashes(first, minimumTokenMatch, isBaseCodeComparison); // don't make table if it is not a base code comparison
-        }
-        if (second.hashLength != minimumTokenMatch || second.tokenHashes == null) {
-            createHashes(second, minimumTokenMatch, true);
-        }
+        SubsequenceHashLookupTable leftLookupTable = subsequenceHashLookupTableForSubmission(leftSubmission, leftMarkedTokens);
+        SubsequenceHashLookupTable rightLookupTable = subsequenceHashLookupTableForSubmission(rightSubmission, rightMarkedTokens);
 
-        List<Match> matches = new ArrayList<>();
-
-        // start the black magic:
-        int maxMatch;
+        int maximumMatchLength;
+        List<Match> globalMatches = new ArrayList<>();
         do {
-            maxMatch = minimumTokenMatch;
-            matches.clear();
-            for (int x = 0; x < first.size() - maxMatch; x++) {
-                List<Integer> hashedTokens = second.tokenHashes.get(first.getToken(x).getHash());
-                if (first.getToken(x).isMarked() || first.getToken(x).getHash() == -1) {
+            maximumMatchLength = minimumMatchLength;
+            List<Match> iterationMatches = new ArrayList<>();
+            for (int leftStartIndex = 0; leftStartIndex < leftTokens.size() - maximumMatchLength; leftStartIndex++) {
+                int leftSubsequenceHash = leftLookupTable.subsequenceHashForStartIndex(leftStartIndex);
+                if (leftMarkedTokens.contains(leftTokens.get(leftStartIndex)) || leftSubsequenceHash == SubsequenceHashLookupTable.NO_HASH) {
                     continue;
                 }
-                inner: for (Integer y : hashedTokens) {
-                    if (second.getToken(y).isMarked() || maxMatch >= second.size() - y) { // >= because of pivots!
+                List<Integer> possiblyMatchingRightStartIndexes = rightLookupTable
+                        .startIndexesOfPossiblyMatchingSubsequencesForSubsequenceHash(leftSubsequenceHash);
+                for (Integer rightStartIndex : possiblyMatchingRightStartIndexes) {
+                    // comparison uses >= because it is assumed that the last token is a pivot (FILE_END)
+                    if (rightMarkedTokens.contains(rightTokens.get(rightStartIndex)) || maximumMatchLength >= rightTokens.size() - rightStartIndex) {
                         continue;
                     }
 
-                    int j, hx, hy;
-                    for (j = maxMatch - 1; j >= 0; j--) { // begins comparison from behind
-                        if (first.getToken(hx = x + j).type != second.getToken(hy = y + j).type || first.getToken(hx).isMarked()
-                                || second.getToken(hy).isMarked()) {
-                            continue inner;
-                        }
+                    if (!subsequencesAreMatchingAndNotMarked(leftTokens.subList(leftStartIndex, leftStartIndex + maximumMatchLength),
+                            leftMarkedTokens, rightTokens.subList(rightStartIndex, rightStartIndex + maximumMatchLength), rightMarkedTokens)) {
+                        continue;
                     }
 
                     // expand match
-                    j = maxMatch;
-                    while (first.getToken(hx = x + j).type == second.getToken(hy = y + j).type && !first.getToken(hx).isMarked()
-                            && !second.getToken(hy).isMarked()) {
-                        j++;
+                    int offset = maximumMatchLength;
+                    while (leftTokens.get(leftStartIndex + offset).type == rightTokens.get(rightStartIndex + offset).type
+                            && !leftMarkedTokens.contains(leftTokens.get(leftStartIndex + offset))
+                            && !rightMarkedTokens.contains(rightTokens.get(rightStartIndex + offset))) {
+                        offset++;
                     }
 
-                    if (j > maxMatch && !isBaseCodeComparison || j != maxMatch && isBaseCodeComparison) {  // new biggest match? -> delete current
-                                                                                                           // smaller
-                        matches.clear();
-                        maxMatch = j;
+                    if (offset > maximumMatchLength) {
+                        iterationMatches.clear();
+                        maximumMatchLength = offset;
                     }
-                    addMatchIfNotOverlapping(matches, x, y, j);
+                    Match match = new Match(leftStartIndex, rightStartIndex, offset);
+                    addMatchIfNotOverlapping(iterationMatches, match);
                 }
             }
-            for (int i = matches.size() - 1; i >= 0; i--) {
-                int x = matches.get(i).startOfFirst();  // Beginning of/in sequence A
-                int y = matches.get(i).startOfSecond();  // Beginning of/in sequence B
-                comparison.addMatch(x, y, matches.get(i).length());
-                // in order that "Match" will be newly build (because reusing)
-                for (int j = matches.get(i).length(); j > 0; j--) {
-                    first.getToken(x).setMarked(true); // mark all Tokens!
-                    second.getToken(y).setMarked(true);
-                    if (isBaseCodeComparison) {
-                        first.getToken(x).setBasecode(true);
-                        second.getToken(y).setBasecode(true);
-                    }
-                    x++;
-                    y++;
+            for (Match match : iterationMatches) {
+                addMatchIfNotOverlapping(globalMatches, match);
+                int leftStartIndex = match.startOfFirst();
+                int rightStartIndex = match.startOfSecond();
+                for (int offset = 0; offset < match.length(); offset++) {
+                    leftMarkedTokens.add(leftTokens.get(leftStartIndex + offset));
+                    rightMarkedTokens.add(rightTokens.get(rightStartIndex + offset));
                 }
             }
+        } while (maximumMatchLength != minimumMatchLength);
+        return new JPlagComparison(leftSubmission, rightSubmission, globalMatches);
+    }
 
-        } while (maxMatch != minimumTokenMatch);
-
-        return comparison;
+    /**
+     * Checks if the two provided subsequences are equal and not marked. Comparison is performed backwards based on the
+     * assumption that the further tokens are away, the more likely they differ. leftTokens and rightTokens must be of equal
+     * size.
+     * @param leftTokens The subsequence of left tokens.
+     * @param leftMarkedTokens The marked tokens of the left token list.
+     * @param rightTokens The subsequence of right tokens.
+     * @param rightMarkedTokens The marked tokens of the right token list.
+     * @return true if the subsequences are matching and not marked, otherwise false.
+     */
+    private boolean subsequencesAreMatchingAndNotMarked(List<Token> leftTokens, Set<Token> leftMarkedTokens, List<Token> rightTokens,
+            Set<Token> rightMarkedTokens) {
+        for (int offset = leftTokens.size() - 1; offset >= 0; offset--) {
+            Token leftToken = leftTokens.get(offset);
+            Token rightToken = rightTokens.get(offset);
+            if (leftToken.type != rightToken.type || leftMarkedTokens.contains(leftToken) || rightMarkedTokens.contains(rightToken)) {
+                return false;
+            }
+        }
+        return true;
     }
 
-    private void addMatchIfNotOverlapping(List<Match> matches, int startA, int startB, int length) {
+    private void addMatchIfNotOverlapping(List<Match> matches, Match match) {
         for (int i = matches.size() - 1; i >= 0; i--) { // starting at the end is better(?)
-            if (matches.get(i).overlap(startA, startB, length)) {
+            if (matches.get(i).overlaps(match)) {
                 return; // no overlaps allowed!
             }
         }
-        matches.add(new Match(startA, startB, length));
+        matches.add(match);
     }
 
-    /**
-     * Disable finding a match at separator tokens and basecode matches for non-basecode comparisons.
-     * @param tokenList Tokens to mark.
-     * @param isBaseCodeComparison Whether the {@link Token#isBasecode()} matches should be enabled for matching.
-     */
-    private void markTokens(TokenList tokenList, boolean isBaseCodeComparison) {
-        for (Token token : tokenList.allTokens()) {
-            if (isBaseCodeComparison) {
-                token.setMarked(token.type == FILE_END || token.type == SEPARATOR_TOKEN);
-            } else {
-                token.setMarked(token.type == FILE_END || token.type == SEPARATOR_TOKEN || (token.isBasecode() && options.hasBaseCode()));
-            }
+    private Set<Token> initiallyMarkedTokens(Submission submission) {
+        Set<Token> baseCodeTokens = baseCodeMarkings.get(submission);
+        return submission.getTokenList().stream().filter(
+                token -> token.type == FILE_END || token.type == SEPARATOR_TOKEN || (baseCodeTokens != null && baseCodeTokens.contains(token)))
+                .collect(Collectors.toSet());
+    }
+
+    private SubsequenceHashLookupTable subsequenceHashLookupTableForSubmission(Submission submission, Set<Token> markedTokens) {
+        if (cachedHashLookupTables.containsKey(submission)) {
+            return cachedHashLookupTables.get(submission);
         }
+        SubsequenceHashLookupTable lookupTable = new SubsequenceHashLookupTable(minimumMatchLength, submission.getTokenList(), markedTokens);
+        cachedHashLookupTables.put(submission, lookupTable);
+        return lookupTable;
     }
 }

jplag/src/main/java/de/jplag/JPlag.java

@@ -45,10 +45,6 @@ public class JPlag {
         SubmissionSetBuilder builder = new SubmissionSetBuilder(language, options);
         SubmissionSet submissionSet = builder.buildSubmissionSet();
 
-        if (submissionSet.hasBaseCode()) {
-            coreAlgorithm.createHashes(submissionSet.getBaseCode().getTokenList(), options.minimumTokenMatch(), true);
-        }
-
         int submissionCount = submissionSet.numberOfSubmissions();
         if (submissionCount < 2) {
             throw new SubmissionException("Not enough valid submissions! (found " + submissionCount + " valid submissions)");

jplag/src/main/java/de/jplag/JPlagComparison.java

@@ -1,103 +1,32 @@
 package de.jplag;
 
-import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
-import java.util.Objects;
 
 /**
- * This method represents the whole result of a comparison between two submissions.
+ * This record represents the whole result of a comparison between two submissions.
+ * @param firstSubmission is the first of the two submissions.
+ * @param secondSubmission is the second of the two submissions.
+ * @param matches is the unmodifiable list of all matches between the two submissions.
  */
-public class JPlagComparison { // FIXME TS: contains a lot of code duplication
-
-    private static final int ROUNDING_FACTOR = 10;
-
-    private final Submission firstSubmission;
-    private final Submission secondSubmission;
-
-    private final List<Match> matches;
-
-    public JPlagComparison(Submission firstSubmission, Submission secondSubmission) {
-        this.firstSubmission = firstSubmission;
-        this.secondSubmission = secondSubmission;
-        matches = new ArrayList<>();
-    }
-
-    /**
-     * Add a match to the comparison (token indices and number of tokens), if it does not overlap with the existing matches.
-     * @see Match#Match(int, int, int)
-     */
-    /* package-private */ final void addMatch(int startOfFirst, int startOfSecond, int length) {
-        for (Match match : matches) {
-            if (match.overlap(startOfFirst, startOfSecond, length)) {
-                return;
-            }
-        }
-        matches.add(new Match(startOfFirst, startOfSecond, length));
-    }
-
-    @Override
-    public boolean equals(Object other) {
-        if (other == this) {
-            return true;
-        }
-        if (!(other instanceof JPlagComparison otherComparison)) {
-            return false;
-        }
-        return firstSubmission.equals(otherComparison.getFirstSubmission()) && secondSubmission.equals(otherComparison.getSecondSubmission())
-                && matches.equals(otherComparison.matches);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(firstSubmission, secondSubmission, matches);
-    }
-
-    /**
-     * @return the base code matches of the first submission.
-     */
-    public JPlagComparison getFirstBaseCodeMatches() {
-        return firstSubmission.getBaseCodeComparison();
-    }
-
-    /**
-     * @return the first of the two submissions.
-     */
-    public Submission getFirstSubmission() {
-        return firstSubmission;
-    }
-
+public record JPlagComparison(Submission firstSubmission, Submission secondSubmission, List<Match> matches) {
     /**
-     * @return all matches between the two submissions.
+     * Initializes a new comparison.
+     * @param firstSubmission is the first of the two submissions.
+     * @param secondSubmission is the second of the two submissions.
+     * @param matches is the list of all matches between the two submissions.
      */
-    public List<Match> getMatches() {
-        return matches;
+    public JPlagComparison(Submission firstSubmission, Submission secondSubmission, List<Match> matches) {
+        this.firstSubmission = firstSubmission;
+        this.secondSubmission = secondSubmission;
+        this.matches = Collections.unmodifiableList(matches);
     }
 
     /**
      * Get the total number of matched tokens for this comparison.
      */
     public final int getNumberOfMatchedTokens() {
-        int numberOfMatchedTokens = 0;
-
-        for (Match match : matches) {
-            numberOfMatchedTokens += match.length();
-        }
-
-        return numberOfMatchedTokens;
-    }
-
-    /**
-     * @return the base code matches of the second submissions.
-     */
-    public JPlagComparison getSecondBaseCodeMatches() {
-        return secondSubmission.getBaseCodeComparison();
-    }
-
-    /**
-     * @return the second of the two submissions.
-     */
-    public Submission getSecondSubmission() {
-        return secondSubmission;
+        return matches.stream().mapToInt(Match::length).sum();
     }
 
     /**
@@ -119,9 +48,9 @@ public class JPlagComparison { // FIXME TS: contains a lot of code duplication
      */
     public final double similarity() {
         boolean subtractBaseCode = firstSubmission.hasBaseCodeMatches() && secondSubmission.hasBaseCodeMatches();
-        double sa = firstSubmission.getSimilarityDivisor(subtractBaseCode);
-        double sb = secondSubmission.getSimilarityDivisor(subtractBaseCode);
-        return (200 * getNumberOfMatchedTokens()) / (sa + sb);
+        int divisorA = firstSubmission.getSimilarityDivisor(subtractBaseCode);
+        int divisorB = secondSubmission.getSimilarityDivisor(subtractBaseCode);
+        return 2 * similarity(divisorA + divisorB);
     }
 
     /**
@@ -130,7 +59,7 @@ public class JPlagComparison { // FIXME TS: contains a lot of code duplication
      */
     public final double similarityOfFirst() {
         int divisor = firstSubmission.getSimilarityDivisor(true);
-        return (divisor == 0 ? 0 : (getNumberOfMatchedTokens() * 100 / (double) divisor));
+        return similarity(divisor);
     }
 
     /**
@@ -139,28 +68,7 @@ public class JPlagComparison { // FIXME TS: contains a lot of code duplication
      */
     public final double similarityOfSecond() {
         int divisor = secondSubmission.getSimilarityDivisor(true);
-        return (divisor == 0 ? 0 : (getNumberOfMatchedTokens() * 100 / (double) divisor));
-    }
-
-    /**
-     * @return Similarity in percent rounded down to the nearest tenth.
-     */
-    public final double roundedSimilarity() {
-        return ((int) (similarity() * ROUNDING_FACTOR)) / (double) ROUNDING_FACTOR;
-    }
-
-    /**
-     * @return Similarity of the first submission to the basecode in percent rounded down to the nearest tenth.
-     */
-    public final double basecodeSimilarityOfFirst() {
-        return ((int) (firstBasecodeSimilarity() * ROUNDING_FACTOR)) / (double) ROUNDING_FACTOR;
-    }
-
-    /**
-     * @return Similarity of the second submission to the basecode in percent rounded down to the nearest tenth.
-     */
-    public final double basecodeSimilarityOfSecond() {
-        return ((int) (secondBasecodeSimilarity() * ROUNDING_FACTOR)) / (double) ROUNDING_FACTOR;
+        return similarity(divisor);
     }
 
     @Override
@@ -168,15 +76,7 @@ public class JPlagComparison { // FIXME TS: contains a lot of code duplication
         return firstSubmission.getName() + " <-> " + secondSubmission.getName();
     }
 
-    private double firstBasecodeSimilarity() {
-        double sa = firstSubmission.getSimilarityDivisor(false);
-        JPlagComparison firstBaseCodeMatches = firstSubmission.getBaseCodeComparison();
-        return firstBaseCodeMatches.getNumberOfMatchedTokens() * 100 / sa;
-    }
-
-    private double secondBasecodeSimilarity() {
-        double sb = secondSubmission.getSimilarityDivisor(false);
-        JPlagComparison secondBaseCodeMatches = secondSubmission.getBaseCodeComparison();
-        return secondBaseCodeMatches.getNumberOfMatchedTokens() * 100 / sb;
+    private double similarity(int divisor) {
+        return (divisor == 0 ? 0.0 : (getNumberOfMatchedTokens() * 100 / (double) divisor));
     }
 }

jplag/src/main/java/de/jplag/Match.java

@@ -11,21 +11,21 @@ public record Match(int startOfFirst, int startOfSecond, int length) {
      * Checks if two matches overlap.
      * @return true if they do.
      */
-    public boolean overlap(int otherStartOfFirst, int otherStartOfSecond, int otherLength) {
-        if (startOfFirst < otherStartOfFirst) {
-            if ((otherStartOfFirst - startOfFirst) < length) {
+    public boolean overlaps(Match other) {
+        if (startOfFirst < other.startOfFirst) {
+            if ((other.startOfFirst - startOfFirst) < length) {
                 return true;
             }
         } else {
-            if ((startOfFirst - otherStartOfFirst) < otherLength) {
+            if ((startOfFirst - other.startOfFirst) < other.length) {
                 return true;
             }
         }
 
-        if (startOfSecond < otherStartOfSecond) {
-            return (otherStartOfSecond - startOfSecond) < length;
+        if (startOfSecond < other.startOfSecond) {
+            return (other.startOfSecond - startOfSecond) < length;
         } else {
-            return (startOfSecond - otherStartOfSecond) < otherLength;
+            return (startOfSecond - other.startOfSecond) < other.length;
         }
     }
 }

jplag/src/main/java/de/jplag/Submission.java

@@ -5,6 +5,8 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 import java.util.Objects;
 
 import org.slf4j.Logger;
@@ -49,7 +51,7 @@ public class Submission implements Comparable<Submission> {
     /**
      * Parse result, tokens from all files.
      */
-    private TokenList tokenList;
+    private List<Token> tokenList;
 
     /**
      * Base code comparison
@@ -138,7 +140,7 @@ public class Submission implements Comparable<Submission> {
      * @param subtractBaseCode If true subtract basecode matches if possible.
      * @return Similarity divisor for the submission.
      */
-    public int getSimilarityDivisor(boolean subtractBaseCode) {
+    int getSimilarityDivisor(boolean subtractBaseCode) {
         int divisor = getNumberOfTokens() - getFiles().size();
         if (subtractBaseCode && baseCodeComparison != null) {
             divisor -= baseCodeComparison.getNumberOfMatchedTokens();
@@ -147,10 +149,10 @@ public class Submission implements Comparable<Submission> {
     }
 
     /**
-     * @return Parse result of the submission.
+     * @return unmodifiable list of tokens generated by parsing the submission.
      */
-    public TokenList getTokenList() {
-        return tokenList;
+    public List<Token> getTokenList() {
+        return tokenList == null ? null : Collections.unmodifiableList(tokenList);
     }
 
     /**
@@ -174,15 +176,6 @@ public class Submission implements Comparable<Submission> {
         return isNew;
     }
 
-    /**
-     * Resets the base code flag for all tokens of this submission.
-     */
-    public void resetBaseCode() {
-        for (Token token : tokenList.allTokens()) {
-            token.setBasecode(false);
-        }
-    }
-
     /**
      * Sets the base code comparison
      * @param baseCodeComparison is submissions matches with the base code
@@ -195,7 +188,7 @@ public class Submission implements Comparable<Submission> {
      * Sets the tokens that have been parsed from the files this submission consists of.
      * @param tokenList is the list of these tokens.
      */
-    public void setTokenList(TokenList tokenList) {
+    public void setTokenList(List<Token> tokenList) {
         this.tokenList = tokenList;
     }
 

jplag/src/main/java/de/jplag/SubsequenceHashLookupTable.java

@@ -0,0 +1,126 @@
+package de.jplag;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A class to generate and store hashes over a fixed length subsequence of a given list of tokens. Hash generation is
+ * optimized to work in O(n).
+ */
+class SubsequenceHashLookupTable {
+    /**
+     * Value combination is chosen such that the maximum possible hash value does not exceed Int.max. Computation formula
+     * for maximum hash value is \sum from (i=0 to MAX_HASH_LENGTH - 1) with (TOKEN_HASH_MODULO - 1) * 2^i
+     */
+    private static final int MAX_HASH_LENGTH = 25;
+    private static final int TOKEN_HASH_MODULO = 64;
+
+    /** Indicator that the subsequence should not be considered for comparison matching */
+    public static final int NO_HASH = -1;
+
+    private final int windowSize;
+    private final List<Token> tokens;
+    private int[] subsequenceHashes;
+    private Map<Integer, List<Integer>> startIndexToSubsequenceHashesMap;
+
+    /**
+     * Generates a new subsequence hash lookup table. Performance is optimized to compute hashes in O(n).
+     * @param windowSize the size of the subsequences.
+     * @param tokens the tokens to hash over.
+     * @param markedTokens the set of marked tokens. Subsequences containing a marked token obtain the {@link #NO_HASH}
+     * value.
+     */
+    SubsequenceHashLookupTable(int windowSize, List<Token> tokens, Set<Token> markedTokens) {
+        windowSize = Math.max(1, windowSize);
+        windowSize = Math.min(MAX_HASH_LENGTH, windowSize);
+        this.windowSize = windowSize;
+        this.tokens = tokens;
+
+        if (tokens.size() < windowSize) {
+            return;
+        }
+
+        subsequenceHashes = new int[tokens.size() - windowSize];
+        startIndexToSubsequenceHashesMap = new HashMap<>(subsequenceHashes.length);
+        computeSubsequenceHashes(markedTokens);
+    }
+
+    /** Returns the size of the subsequences used for hashing */
+    int getWindowSize() {
+        return windowSize;
+    }
+
+    /** Returns the list of tokens for which the hashes were computed */
+    List<Token> getTokens() {
+        return tokens;
+    }
+
+    /**
+     * Returns the hash over the subsequence from startIndex to startIndex+windowSize.
+     * @param startIndex the start index.
+     * @return the hash of the requested subsequence.
+     */
+    int subsequenceHashForStartIndex(int startIndex) {
+        return subsequenceHashes[startIndex];
+    }
+
+    /**
+     * Returns a list of all start indexes of possible subsequences for the given subsequence hash.
+     * @param subsequenceHash the hash value to obtain possibly matching subsequence start indexes for.
+     * @return a list with possible matching start indexes.
+     */
+    List<Integer> startIndexesOfPossiblyMatchingSubsequencesForSubsequenceHash(int subsequenceHash) {
+        if (startIndexToSubsequenceHashesMap.containsKey(subsequenceHash)) {
+            return startIndexToSubsequenceHashesMap.get(subsequenceHash);
+        }
+        return List.of();
+    }
+
+    /**
+     * Creates hashes for all subsequences with windowSize. Code is optimized to perform in O(n) using a windowing approach.
+     * Hashes are computed by \sum from (i=0 to windowSize) with hash(tokens[offset+i]) * 2^(hashLength-1-i)
+     * @param markedTokens contains the marked tokens. Subsequences containing a marked token will receive the NO_HASH
+     * value.
+     */
+    private void computeSubsequenceHashes(Set<Token> markedTokens) {
+        int hash = 0;
+        int hashedLength = 0;
+        int factor = (windowSize != 1 ? (2 << (windowSize - 2)) : 1);
+
+        for (int windowEndIndex = 0; windowEndIndex < tokens.size(); windowEndIndex++) {
+            int windowStartIndex = windowEndIndex - windowSize;
+            if (windowStartIndex >= 0) {
+                if (hashedLength >= windowSize) {
+                    subsequenceHashes[windowStartIndex] = hash;
+                    addToStartIndexesToHashesMap(windowStartIndex, hash);
+                } else {
+                    subsequenceHashes[windowStartIndex] = NO_HASH;
+                }
+                hash -= factor * (hashValueForToken(tokens.get(windowStartIndex)));
+            }
+            hash = (2 * hash) + (hashValueForToken(tokens.get(windowEndIndex)));
+            if (markedTokens.contains(tokens.get(windowEndIndex))) {
+                hashedLength = 0;
+            } else {
+                hashedLength++;
+            }
+        }
+    }
+
+    private int hashValueForToken(Token token) {
+        return token.type % TOKEN_HASH_MODULO;
+    }
+
+    private void addToStartIndexesToHashesMap(int startIndex, int subsequenceHash) {
+        if (startIndexToSubsequenceHashesMap.containsKey(subsequenceHash)) {
+            startIndexToSubsequenceHashesMap.get(subsequenceHash).add(startIndex);
+        } else {
+            List<Integer> startIndexes = new ArrayList<>();
+            startIndexes.add(startIndex);
+            startIndexToSubsequenceHashesMap.put(subsequenceHash, startIndexes);
+        }
+    }
+}

jplag/src/main/java/de/jplag/clustering/ClusteringAdapter.java

@@ -33,15 +33,15 @@ public class ClusteringAdapter {
     public ClusteringAdapter(Collection<JPlagComparison> comparisons, ToDoubleFunction<JPlagComparison> metric) {
         mapping = new IntegerMapping<>(comparisons.size());
         for (JPlagComparison comparison : comparisons) {
-            mapping.map(comparison.getFirstSubmission());
-            mapping.map(comparison.getSecondSubmission());
+            mapping.map(comparison.firstSubmission());
+            mapping.map(comparison.secondSubmission());
         }
         int size = mapping.size();
 
         similarityMatrix = new Array2DRowRealMatrix(size, size);
         for (JPlagComparison comparison : comparisons) {
-            int firstIndex = mapping.map(comparison.getFirstSubmission());
-            int secondIndex = mapping.map(comparison.getSecondSubmission());
+            int firstIndex = mapping.map(comparison.firstSubmission());
+            int secondIndex = mapping.map(comparison.secondSubmission());
             double similarity = metric.applyAsDouble(comparison);
             similarityMatrix.setEntry(firstIndex, secondIndex, similarity);
             similarityMatrix.setEntry(secondIndex, firstIndex, similarity);

jplag/src/main/java/de/jplag/reporting/jsonfactory/ComparisonReportWriter.java

@@ -9,7 +9,6 @@ import de.jplag.JPlagComparison;
 import de.jplag.JPlagResult;
 import de.jplag.Submission;
 import de.jplag.Token;
-import de.jplag.TokenList;
 import de.jplag.reporting.reportobject.model.ComparisonReport;
 import de.jplag.reporting.reportobject.model.Match;
 
@@ -47,8 +46,8 @@ public class ComparisonReportWriter {
 
     private void writeComparisons(String path, List<JPlagComparison> comparisons) {
         comparisons.parallelStream().forEach(comparison -> {
-            String firstSubmissionId = submissionToIdFunction.apply(comparison.getFirstSubmission());
-            String secondSubmissionId = submissionToIdFunction.apply(comparison.getSecondSubmission());
+            String firstSubmissionId = submissionToIdFunction.apply(comparison.firstSubmission());
+            String secondSubmissionId = submissionToIdFunction.apply(comparison.secondSubmission());
             String fileName = generateComparisonName(firstSubmissionId, secondSubmissionId);
             addToLookUp(firstSubmissionId, secondSubmissionId, fileName);
             var comparisonReport = new ComparisonReport(firstSubmissionId, secondSubmissionId, comparison.similarity(),
@@ -87,16 +86,16 @@ public class ComparisonReportWriter {
     }
 
     private List<Match> convertMatchesToReportMatches(JPlagComparison comparison) {
-        return comparison.getMatches().stream().map(match -> convertMatchToReportMatch(comparison, match)).toList();
+        return comparison.matches().stream().map(match -> convertMatchToReportMatch(comparison, match)).toList();
     }
 
     private Match convertMatchToReportMatch(JPlagComparison comparison, de.jplag.Match match) {
-        TokenList tokensFirst = comparison.getFirstSubmission().getTokenList();
-        TokenList tokensSecond = comparison.getSecondSubmission().getTokenList();
-        Token startOfFirst = tokensFirst.getToken(match.startOfFirst());
-        Token endOfFirst = tokensFirst.getToken(match.startOfFirst() + match.length() - 1);
-        Token startOfSecond = tokensSecond.getToken(match.startOfSecond());
-        Token endOfSecond = tokensSecond.getToken(match.startOfSecond() + match.length() - 1);
+        List<Token> tokensFirst = comparison.firstSubmission().getTokenList();
+        List<Token> tokensSecond = comparison.secondSubmission().getTokenList();
+        Token startOfFirst = tokensFirst.get(match.startOfFirst());
+        Token endOfFirst = tokensFirst.get(match.startOfFirst() + match.length() - 1);
+        Token startOfSecond = tokensSecond.get(match.startOfSecond());
+        Token endOfSecond = tokensSecond.get(match.startOfSecond() + match.length() - 1);
 
         return new Match(startOfFirst.getFile(), startOfSecond.getFile(), startOfFirst.getLine(), endOfFirst.getLine(), startOfSecond.getLine(),
                 endOfSecond.getLine(), match.length());

jplag/src/main/java/de/jplag/reporting/reportobject/ReportObjectFactory.java

@@ -160,8 +160,8 @@ public class ReportObjectFactory {
     }
 
     private Set<Submission> getSubmissions(List<JPlagComparison> comparisons) {
-        Set<Submission> submissions = comparisons.stream().map(JPlagComparison::getFirstSubmission).collect(Collectors.toSet());
-        Set<Submission> secondSubmissions = comparisons.stream().map(JPlagComparison::getSecondSubmission).collect(Collectors.toSet());
+        Set<Submission> submissions = comparisons.stream().map(JPlagComparison::firstSubmission).collect(Collectors.toSet());
+        Set<Submission> secondSubmissions = comparisons.stream().map(JPlagComparison::secondSubmission).collect(Collectors.toSet());
         submissions.addAll(secondSubmissions);
         return submissions;
     }

jplag/src/main/java/de/jplag/reporting/reportobject/mapper/MetricMapper.java

@@ -45,8 +45,8 @@ public class MetricMapper {
 
     private List<TopComparison> getTopComparisons(List<JPlagComparison> comparisons, Function<JPlagComparison, Double> similarityExtractor) {
         return comparisons.stream().sorted(Comparator.comparing(similarityExtractor).reversed())
-                .map(comparison -> new TopComparison(submissionToIdFunction.apply(comparison.getFirstSubmission()),
-                        submissionToIdFunction.apply(comparison.getSecondSubmission()), similarityExtractor.apply(comparison)))
+                .map(comparison -> new TopComparison(submissionToIdFunction.apply(comparison.firstSubmission()),
+                        submissionToIdFunction.apply(comparison.secondSubmission()), similarityExtractor.apply(comparison)))
                 .toList();
     }
 

jplag/src/main/java/de/jplag/reporting/reportobject/mapper/SubmissionNameToIdMapper.java

@@ -26,8 +26,8 @@ public class SubmissionNameToIdMapper {
     public static Map<String, String> buildSubmissionNameToIdMap(JPlagResult result) {
         HashMap<String, String> idToName = new HashMap<>();
         getComparisons(result).forEach(comparison -> {
-            idToName.put(comparison.getFirstSubmission().getName(), sanitizeNameOf(comparison.getFirstSubmission()));
-            idToName.put(comparison.getSecondSubmission().getName(), sanitizeNameOf(comparison.getSecondSubmission()));
+            idToName.put(comparison.firstSubmission().getName(), sanitizeNameOf(comparison.firstSubmission()));
+            idToName.put(comparison.secondSubmission().getName(), sanitizeNameOf(comparison.secondSubmission()));
         });
         return idToName;
     }

jplag/src/main/java/de/jplag/strategy/AbstractComparisonStrategy.java

@@ -36,9 +36,8 @@ public abstract class AbstractComparisonStrategy implements ComparisonStrategy {
     protected void compareSubmissionsToBaseCode(SubmissionSet submissionSet) {
         Submission baseCodeSubmission = submissionSet.getBaseCode();
         for (Submission currentSubmission : submissionSet.getSubmissions()) {
-            JPlagComparison baseCodeComparison = greedyStringTiling.compareWithBaseCode(currentSubmission, baseCodeSubmission);
+            JPlagComparison baseCodeComparison = greedyStringTiling.generateBaseCodeMarking(currentSubmission, baseCodeSubmission);
             currentSubmission.setBaseCodeComparison(baseCodeComparison);
-            baseCodeSubmission.resetBaseCode();
         }
     }
 

jplag/src/test/java/de/jplag/BaseCodeTest.java

@@ -40,7 +40,7 @@ public class BaseCodeTest extends TestBase {
     private void verifyResults(JPlagResult result) {
         assertEquals(2, result.getNumberOfSubmissions());
         assertEquals(1, result.getAllComparisons().size());
-        assertEquals(1, result.getAllComparisons().get(0).getMatches().size());
+        assertEquals(1, result.getAllComparisons().get(0).matches().size());
         assertEquals(1, result.getSimilarityDistribution()[1]);
         assertEquals(85, result.getAllComparisons().get(0).similarity(), DELTA);
     }

jplag/src/test/java/de/jplag/NewJavaFeaturesTest.java

@@ -28,7 +28,7 @@ public class NewJavaFeaturesTest extends TestBase {
         // Check similarity and number of matches:
         var comparison = result.getAllComparisons().get(0);
         assertEquals(EXPECTED_SIMILARITY, comparison.similarity(), DELTA);
-        assertEquals(EXPECTED_MATCHES, comparison.getMatches().size());
+        assertEquals(EXPECTED_MATCHES, comparison.matches().size());
     }
 
 }

jplag/src/test/java/de/jplag/NormalComparisonTest.java

@@ -23,7 +23,7 @@ class NormalComparisonTest extends TestBase {
 
         assertEquals(2, result.getNumberOfSubmissions());
         assertEquals(1, result.getAllComparisons().size());
-        assertEquals(1, result.getAllComparisons().get(0).getMatches().size());
+        assertEquals(1, result.getAllComparisons().get(0).matches().size());
         assertEquals(1, result.getSimilarityDistribution()[3]);
         assertEquals(62.07, result.getAllComparisons().get(0).similarity(), 0.1);
     }
@@ -38,7 +38,7 @@ class NormalComparisonTest extends TestBase {
 
         assertEquals(2, result.getNumberOfSubmissions());
         assertEquals(1, result.getAllComparisons().size());
-        assertEquals(2, result.getAllComparisons().get(0).getMatches().size());
+        assertEquals(2, result.getAllComparisons().get(0).matches().size());
         assertArrayEquals(expectedDistribution, result.getSimilarityDistribution());
         assertEquals(96.55, result.getAllComparisons().get(0).similarity(), 0.1);
     }
@@ -70,7 +70,7 @@ class NormalComparisonTest extends TestBase {
 
         // All comparisons with E shall have no matches
         result.getAllComparisons().stream()
-                .filter(comparison -> comparison.getSecondSubmission().getName().equals("E") || comparison.getFirstSubmission().getName().equals("E"))
+                .filter(comparison -> comparison.secondSubmission().getName().equals("E") || comparison.firstSubmission().getName().equals("E"))
                 .forEach(comparison -> assertEquals(0, comparison.similarity(), DELTA));
 
         // Hard coded assertions on selected comparisons
@@ -85,7 +85,7 @@ class NormalComparisonTest extends TestBase {
         var biggestMatch = getSelectedComparison(result, "A", "D");
         assertEquals(96.4, biggestMatch.get().maximalSimilarity(), 0.1);
         assertEquals(65.3, biggestMatch.get().minimalSimilarity(), 0.1);
-        assertEquals(12, biggestMatch.get().getMatches().size());
+        assertEquals(12, biggestMatch.get().matches().size());
 
     }
 
@@ -95,9 +95,9 @@ class NormalComparisonTest extends TestBase {
     }
 
     private Optional<JPlagComparison> getSelectedComparison(JPlagResult result, String nameA, String nameB) {
-        return result.getAllComparisons().stream().filter(
-                comparison -> comparison.getFirstSubmission().getName().equals(nameA) && comparison.getSecondSubmission().getName().equals(nameB)
-                        || comparison.getFirstSubmission().getName().equals(nameB) && comparison.getSecondSubmission().getName().equals(nameA))
+        return result.getAllComparisons().stream()
+                .filter(comparison -> comparison.firstSubmission().getName().equals(nameA) && comparison.secondSubmission().getName().equals(nameB)
+                        || comparison.firstSubmission().getName().equals(nameB) && comparison.secondSubmission().getName().equals(nameA))
                 .findFirst();
     }
 

jplag/src/test/java/de/jplag/ParallelComparisonTest.java

@@ -26,7 +26,7 @@ public class ParallelComparisonTest extends TestBase {
 
         assertEquals(2, result.getNumberOfSubmissions());
         assertEquals(1, result.getAllComparisons().size());
-        assertEquals(1, result.getAllComparisons().get(0).getMatches().size());
+        assertEquals(1, result.getAllComparisons().get(0).matches().size());
         assertEquals(1, result.getSimilarityDistribution()[3]);
         assertEquals(62.07, result.getAllComparisons().get(0).similarity(), DELTA);
     }
@@ -58,7 +58,7 @@ public class ParallelComparisonTest extends TestBase {
 
         // All comparisons with E shall have no matches
         result.getAllComparisons().stream()
-                .filter(comparison -> comparison.getSecondSubmission().getName().equals("E") || comparison.getFirstSubmission().getName().equals("E"))
+                .filter(comparison -> comparison.secondSubmission().getName().equals("E") || comparison.firstSubmission().getName().equals("E"))
                 .forEach(comparison -> assertEquals(0, comparison.similarity(), DELTA));
 
         // Hard coded assertions on selected comparisons
@@ -73,8 +73,7 @@ public class ParallelComparisonTest extends TestBase {
         var biggestMatch = getSelectedComparison(result, "A", "D");
         assertEquals(96.4, biggestMatch.get().maximalSimilarity(), DELTA);
         assertEquals(65.3, biggestMatch.get().minimalSimilarity(), DELTA);
-        assertEquals(12, biggestMatch.get().getMatches().size());
-
+        assertEquals(12, biggestMatch.get().matches().size());
     }
 
     // TODO SH: Methods like this should be moved to the API and also should accept wildcards
@@ -83,9 +82,9 @@ public class ParallelComparisonTest extends TestBase {
     }
 
     private Optional<JPlagComparison> getSelectedComparison(JPlagResult result, String nameA, String nameB) {
-        return result.getAllComparisons().stream().filter(
-                comparison -> comparison.getFirstSubmission().getName().equals(nameA) && comparison.getSecondSubmission().getName().equals(nameB)
-                        || comparison.getFirstSubmission().getName().equals(nameB) && comparison.getSecondSubmission().getName().equals(nameA))
+        return result.getAllComparisons().stream()
+                .filter(comparison -> comparison.firstSubmission().getName().equals(nameA) && comparison.secondSubmission().getName().equals(nameB)
+                        || comparison.firstSubmission().getName().equals(nameB) && comparison.secondSubmission().getName().equals(nameA))
                 .findFirst();
     }
 }

jplag/src/test/java/de/jplag/clustering/ClusteringAdapterTest.java

@@ -28,8 +28,8 @@ public class ClusteringAdapterTest {
         for (int i = 0; i < submissions.size(); i++) {
             for (int j = i + 1; j < submissions.size(); j++) {
                 JPlagComparison comparison = mock(JPlagComparison.class);
-                when(comparison.getFirstSubmission()).thenReturn(submissions.get(i));
-                when(comparison.getSecondSubmission()).thenReturn(submissions.get(j));
+                when(comparison.firstSubmission()).thenReturn(submissions.get(i));
+                when(comparison.secondSubmission()).thenReturn(submissions.get(j));
                 comparisons.add(comparison);
             }
         }

jplag/src/test/java/de/jplag/reporting/reportobject/mapper/MetricMapperTest.java

@@ -89,8 +89,8 @@ public class MetricMapperTest {
             doReturn(comparisonDto.submission2.name).when(submission2).getName();
 
             JPlagComparison mockedComparison = mock(JPlagComparison.class);
-            doReturn(submission1).when(mockedComparison).getFirstSubmission();
-            doReturn(submission2).when(mockedComparison).getSecondSubmission();
+            doReturn(submission1).when(mockedComparison).firstSubmission();
+            doReturn(submission2).when(mockedComparison).secondSubmission();
             if (metricToMock.equals(MockMetric.AVG)) {
                 doReturn(comparisonDto.similarity).when(mockedComparison).similarity();
             } else if (metricToMock.equals(MockMetric.MAX)) {

jplag/src/test/java/de/jplag/special/VolumeTest.java

@@ -55,7 +55,7 @@ public class VolumeTest extends TestBase {
         System.out.println("Volume test size: " + csv.size());
 
         results.getAllComparisons().forEach(result -> {
-            var key = result.getFirstSubmission().getName() + result.getSecondSubmission().getName();
+            var key = result.firstSubmission().getName() + result.secondSubmission().getName();
 
             assertTrue(csv.containsKey(key));
             assertEquals(csv.getOrDefault(key, -1.0), result.similarity(), DELTA);