Merge branch 'master' into feature/general-improvements

# Conflicts:
#	jplag.frontend-utils/src/main/java/de/jplag/Language.java
#	jplag.frontend-utils/src/main/java/de/jplag/Token.java
#	jplag.frontend-utils/src/main/java/de/jplag/TokenConstants.java
#	jplag.frontend.chars/src/main/java/de/jplag/chars/CharToken.java
#	jplag.frontend.chars/src/main/java/de/jplag/chars/Parser.java
#	jplag.frontend.cpp/src/main/java/de/jplag/cpp/CPPToken.java
#	jplag.frontend.cpp/src/main/java/de/jplag/cpp/CPPTokenConstants.java
#	jplag.frontend.cpp/src/main/java/de/jplag/cpp/Language.java
#	jplag.frontend.cpp/src/main/java/de/jplag/cpp/Scanner.java
#	jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/CSharpToken.java
#	jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/CSharpTokenConstants.java
#	jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/Parser.java
#	jplag.frontend.java/src/main/java/de/jplag/java/JavaToken.java
#	jplag.frontend.java/src/main/java/de/jplag/java/JavaTokenConstants.java
#	jplag.frontend.java/src/main/java/de/jplag/java/Parser.java
#	jplag.frontend.python-3/src/main/java/de/jplag/python3/Parser.java
#	jplag.frontend.python-3/src/main/java/de/jplag/python3/Python3Token.java
#	jplag.frontend.python-3/src/main/java/de/jplag/python3/Python3TokenConstants.java
#	jplag.frontend.scheme/src/main/java/de/jplag/scheme/Parser.java
#	jplag.frontend.scheme/src/main/java/de/jplag/scheme/SchemeToken.java
#	jplag.frontend.scheme/src/main/java/de/jplag/scheme/SchemeTokenConstants.java
#	jplag.frontend.text/src/main/java/de/jplag/text/Parser.java
#	jplag/src/main/java/de/jplag/CommandLineArgument.java
#	jplag/src/main/java/de/jplag/options/SimilarityMetric.java

"Resolved conflicts using theirs"

.github/dependabot.yml

@@ -23,4 +23,16 @@ updates:
     labels:
       - "dependencies"
       - "minor"
+  - package-ecosystem: "npm"
+    directory: "/report-viewer"
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+      time: "11:00"
+      timezone: "Europe/Berlin"
+    commit-message:
+      prefix: "Dependency"
+      include: "scope"
+
+
 

.github/workflows/report-viewer.yml

@@ -0,0 +1,30 @@
+name: Report Viewer Deployment Workflow
+
+on:
+ workflow_dispatch: # Use this to dispatch from the Actions Tab
+ push:
+    branches:
+      - master
+      
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout 🛎️
+        uses: actions/checkout@v2.3.5
+      
+      - uses: actions/setup-node@v2
+        with:
+          node-version: "16"
+          
+      - name: Install and Build 🔧
+        working-directory: report-viewer
+        run: |
+          npm install
+          npm run build
+
+      - name: Deploy 🚀
+        uses: JamesIves/github-pages-deploy-action@4.1.5
+        with:
+          branch: gh-pages
+          folder: report-viewer/dist

jplag.frontend-utils/src/main/java/de/jplag/AbstractParser.java

@@ -6,18 +6,18 @@ package de.jplag;
 public abstract class AbstractParser {
     protected ErrorConsumer errorConsumer;
     protected int errors = 0;
-    private int errorsSum = 0;
+    private int numberOfErrors = 0;
 
     public boolean hasErrors() {
         return errors != 0;
     }
 
     public int errorsCount() {
-        return errorsSum;
+        return numberOfErrors;
     }
 
     protected void parseEnd() {
-        errorsSum += errors;
+        numberOfErrors += errors;
     }
 
     public ErrorConsumer getErrorConsumer() {

jplag.frontend-utils/src/main/java/de/jplag/ErrorConsumer.java

@@ -1,5 +1,8 @@
 package de.jplag;
 
+/**
+ * Consumes errors from a language parser.
+ */
 public interface ErrorConsumer {
 
     /**

jplag.frontend-utils/src/main/java/de/jplag/Language.java

@@ -10,60 +10,55 @@ public interface Language {
     /**
      * Suffixes for the files containing code of the language.
      */
-    String[] suffixes();
+    public String[] suffixes();
 
     /**
      * Descriptive name of the language.
      */
-    String getName();
+    public String getName();
 
     /**
      * Short name of the language used for CLI options.
      */
-    String getShortName();
+    public String getShortName();
 
     /**
      * Minimum number of tokens required for a match.
      */
-    int minimumTokenMatch();
+    public int minimumTokenMatch();
 
     /**
      * Parses a set files in a directory.
      */
-    TokenList parse(File dir, String[] files);
+    public TokenList parse(File dir, String[] files);
 
     /**
      * Whether errors were found during the last {@link #parse}.
      */
-    boolean hasErrors();
+    public boolean hasErrors();
 
     /**
      * Number of errors found during the last {@link #parse}.
      */
-    int errorCount();
+    public int errorCount();
 
     /**
      * Determines whether the parser provide column information.
      */
-    boolean supportsColumns();
+    public boolean supportsColumns();
 
     /**
      * Determines whether JPlag should use a fixed-width font in its reports.
      */
-    boolean isPreformatted();
+    public boolean isPreformatted();
 
     /**
      * Determines whether tokens from the scanner are indexed.
      */
-    boolean usesIndex();
+    public boolean usesIndex();
 
     /**
      * Number of defined tokens in the scanner of the language.
      */
-    int numberOfTokens();
-
-    /**
-     * Convert a token type to a text representation.
-     */
-    String type2string(int type);
+    public int numberOfTokens();
 }

jplag.frontend-utils/src/main/java/de/jplag/Token.java

@@ -1,44 +1,54 @@
 package de.jplag;
 
-import java.io.Serial;
-import java.io.Serializable;
+/**
+ * This class represents a token in a source code. It can represents keywords, identifies, syntactical structures etc.
+ * What types of tokens there are depends on the specific language, meaning JPlag does not enforce a specific token set.
+ * The language parsers decide what is a token and what is not.
+ */
+public abstract class Token {
+    private int line;
+    private int column;
+    private int length;
+    private String file;
 
-public abstract class Token implements TokenConstants, Serializable {
-    @Serial
-    private static final long serialVersionUID = 862935679966383302L;
-    public int type;
-    public String file;
+    private boolean marked;
+    private boolean basecode = false;
+    private int hash = -1;// hash-value. set and used by main algorithm (GSTiling)
 
-    protected boolean marked;
-    protected boolean basecode = false;
-    protected int hash = -1;// hash-value. set and used by main algorithm (GSTiling)
+    protected int type;
 
+    /**
+     * Creates a token without information about the column or the length of the token in the line.
+     * @param type is the token type.
+     * @param file is the name of the source code file.
+     * @param line is the line index in the source code where the token resides. Cannot be smaller than 1.
+     */
     public Token(int type, String file, int line) {
-        this(type, file, line, -1, -1);
-    }
-
-    public Token(int type, String file, int line, int column, int length) {
         this.type = type;
         this.file = file;
         setLine(line > 0 ? line : 1);
-        setColumn(column);
-        setLength(length);
     }
 
-    // abstract members
-    public abstract int getLine();
-
-    public abstract int getColumn();
-
-    public abstract int getLength();
-
-    protected void setLine(int line) {
+    /**
+     * Creates a token with column and length information.
+     * @param type is the token type.
+     * @param file is the name of the source code file.
+     * @param line is the line index in the source code where the token resides. Cannot be smaller than 1.
+     * @param column is the column index, meaning where the token starts in the line.
+     * @param length is the length of the token in the source code.
+     */
+    public Token(int type, String file, int line, int column, int length) {
+        this(type, file, line);
+        this.column = column;
+        this.length = length;
     }
 
-    protected void setColumn(int line) {
+    public String getFile() {
+        return file;
     }
 
-    protected void setLength(int line) {
+    public int getHash() {
+        return hash;
     }
 
     // this is made to distinguish the character front end. Maybe other front ends can use it too?
@@ -46,12 +56,60 @@ public abstract class Token implements TokenConstants, Serializable {
         return -1;
     }
 
-    public static String type2string(int type) {
-        return "<abstract>";
+    public int getLine() {
+        return line;
+    }
+
+    public int getColumn() {
+        return column;
+    }
+
+    public int getLength() {
+        return length;
+    }
+
+    public boolean isBasecode() {
+        return basecode;
+    }
+
+    public boolean isMarked() {
+        return marked;
+    }
+
+    public void setLine(int line) {
+        this.line = line;
+    }
+
+    public void setColumn(int column) {
+        this.column = column;
+    }
+
+    public void setLength(int length) {
+        this.length = length;
     }
 
     @Override
     public String toString() {
-        return type2string(type);
+        return type2string();
+    }
+
+    protected abstract String type2string();
+
+    /* Package Private */ boolean setBasecode(boolean basecode) {
+        this.basecode = basecode;
+        return basecode;
+    }
+
+    /* Package Private */ void setFile(String file) {
+        this.file = file;
+    }
+
+    /* Package Private */ void setHash(int hash) {
+        this.hash = hash;
+    }
+
+    /* Package Private */ boolean setMarked(boolean marked) {
+        this.marked = marked;
+        return marked;
     }
 }

jplag.frontend-utils/src/main/java/de/jplag/TokenConstants.java

@@ -1,9 +1,16 @@
 package de.jplag;
 
+/**
+ * Super interface for all language parsers. Provides the only two token that are required from every parser.
+ */
 public interface TokenConstants {
-    int FILE_END = 0;
+    /**
+     * Marks the end of the file, has a special purpose in the comparison algorithm.
+     */
+    final static int FILE_END = 0;
+
     /**
      * Used to optionally separate methods from each other with an always marked token
      */
-    int SEPARATOR_TOKEN = 1;
+    final static int SEPARATOR_TOKEN = 1;
 }

jplag.frontend-utils/src/main/java/de/jplag/TokenList.java

@@ -6,7 +6,7 @@ import java.util.List;
 /**
  * List of tokens. Allows random access to individual tokens. Contains a hash map for token hashes.
  */
-public class TokenList implements TokenConstants {
+public class TokenList {
     private final List<Token> tokens;
     TokenHashMap tokenHashes = null;
     int hash_length = -1;
@@ -22,10 +22,10 @@ public class TokenList implements TokenConstants {
     public final void addToken(Token token) {
         if (tokens.size() > 0) {
             Token lastToken = tokens.get(tokens.size() - 1);
-            if (lastToken.file.equals(token.file)) {
-                token.file = lastToken.file; // To save memory ...
+            if (lastToken.getFile().equals(token.getFile())) {
+                token.setFile(lastToken.getFile()); // To save memory ...
             }
-            if (token.getLine() < lastToken.getLine() && (token.file.equals(lastToken.file))) {
+            if (token.getLine() < lastToken.getLine() && (token.getFile().equals(lastToken.getFile()))) {
                 token.setLine(lastToken.getLine()); // just to make sure
             }
         }

jplag.frontend.chars/src/main/java/de/jplag/chars/CharToken.java

@@ -1,22 +1,20 @@
 package de.jplag.chars;
 
-import java.io.Serial;
-
 import de.jplag.Token;
 
 public class CharToken extends Token {
 
-    @Serial
-    private static final long serialVersionUID = -754166753275082477L;
-    private final int index;
+    private int index;
+    private Character character;
 
-    public CharToken(int type, String file, int index, Parser parser) {
+    public CharToken(int type, String file, int index, Character character, Parser parser) {
         super(type, file, -1);
         this.index = index;
+        this.character = character;
     }
 
     public CharToken(int type, String file, Parser parser) {
-        this(type, file, -1, parser);
+        this(type, file, -1, null, parser);
     }
 
     @Override
@@ -25,17 +23,30 @@ public class CharToken extends Token {
     }
 
     @Override
-    public int getColumn() {
-        return 0;
+    public int getIndex() {
+        return index;
     }
 
     @Override
-    public int getLength() {
-        return 0;
+    public void setLine(int line) {
+        // Do nothing!
     }
 
     @Override
-    public int getIndex() {
-        return index;
+    public void setColumn(int column) {
+        // Do nothing!
+    }
+
+    @Override
+    public void setLength(int length) {
+        // Do nothing!
+    }
+
+    @Override
+    protected String type2string() {
+        if (character == null) {
+            return "FILE_END";
+        }
+        return character.toString();
     }
 }

jplag.frontend.chars/src/main/java/de/jplag/chars/Language.java

@@ -3,7 +3,6 @@ package de.jplag.chars;
 import java.io.File;
 
 import de.jplag.ErrorConsumer;
-import de.jplag.Token;
 import de.jplag.TokenList;
 
 /*
@@ -29,7 +28,7 @@ public class Language implements de.jplag.Language {
 
     @Override
     public String getName() {
-        return "Character AbstractParser";
+        return "Character Parser";
     }
 
     @Override
@@ -71,9 +70,4 @@ public class Language implements de.jplag.Language {
     public int numberOfTokens() {
         return 36;
     }
-
-    @Override
-    public String type2string(int type) {
-        return Token.type2string(type);
-    }
 }

jplag.frontend.chars/src/main/java/de/jplag/chars/Parser.java

@@ -8,18 +8,18 @@ import de.jplag.AbstractParser;
 import de.jplag.TokenConstants;
 import de.jplag.TokenList;
 
-public class Parser extends AbstractParser implements TokenConstants {
+public class Parser extends AbstractParser {
 
-    private TokenList struct;
+    private TokenList tokens;
 
-    public TokenList parse(File dir, String[] files) {
-        struct = new TokenList();
+    public TokenList parse(File directory, String files[]) {
+        tokens = new TokenList();
         errors = 0;
         for (String file : files) {
             getErrorConsumer().print(null, "Parsing file " + file);
-            if (!parseFile(dir, file))
+            if (!parseFile(directory, file))
                 errors++;
-            struct.addToken(new CharToken(FILE_END, file, this));
+            tokens.addToken(new CharToken(TokenConstants.FILE_END, file, this));
         }
         if (errors == 0)
             errorConsumer.print(null, "OK");
@@ -27,7 +27,7 @@ public class Parser extends AbstractParser implements TokenConstants {
             errorConsumer.print(null, errors + " ERROR" + (errors > 1 ? "S" : ""));
 
         this.parseEnd();
-        return struct;
+        return tokens;
     }
 
     private boolean parseFile(File dir, String file) {
@@ -44,7 +44,7 @@ public class Parser extends AbstractParser implements TokenConstants {
 
                 for (int i = 0; i < length; i++) {
                     if (buffer[i] <= 127 && (type = mapping[buffer[i]]) > 1) {
-                        struct.addToken(new CharToken(type, file, offset + i, this));
+                        tokens.addToken(new CharToken(type, file, offset + i, buffer[i], this));
                     }
                 }
                 offset += length;
@@ -59,7 +59,7 @@ public class Parser extends AbstractParser implements TokenConstants {
         return true;
     }
 
-    private final int[] mapping = {-1, // 0 (nul)
+    private int[] mapping = {-1, // 0 (nul)
             -1, // 1 (soh)
             -1, // 2 (stx)
             -1, // 3 (etx)

jplag.frontend.cpp/src/main/java/de/jplag/cpp/CPPToken.java

@@ -1,42 +1,21 @@
 package de.jplag.cpp;
 
-import java.io.Serial;
-
 import de.jplag.Token;
 
 public class CPPToken extends Token implements CPPTokenConstants {
-    @Serial
-    private static final long serialVersionUID = -8962159131223617619L;
-    private int line;
-
-    public CPPToken(int type, String file, int line) {
-        super(type, file, line);
-    }
-
-    @Override
-    public int getLine() {
-        return line;
-    }
-
-    @Override
-    public void setLine(int line) {
-        this.line = line;
-    }
 
-    @Override
-    public int getColumn() {
-        return 0;
+    public CPPToken(int type, String file, int line, int column, int length) {
+        super(type, file, line, column, length);
     }
 
-    @Override
-    public int getLength() {
-        return 0;
+    public CPPToken(int type, String file) {
+        super(type, file, 1, 0, 0);
     }
 
-    public static String type2string(int type) {
+    protected String type2string() {
         switch (type) {
             case CPPTokenConstants.FILE_END:
-                return "********";
+                return "EOF";
 
             case C_BLOCK_BEGIN:
                 return "BLOCK{";
@@ -162,7 +141,7 @@ public class CPPToken extends Token implements CPPTokenConstants {
                 return "NULL";
 
             default:
-                return "<UNBEKANNT>";
+                return "<UNKNOWN" + type + ">";
         }
     }
 }

jplag.frontend.cpp/src/main/java/de/jplag/cpp/CPPTokenConstants.java

@@ -3,72 +3,67 @@ package de.jplag.cpp;
 import de.jplag.TokenConstants;
 
 public interface CPPTokenConstants extends TokenConstants {
-    int FILE_END = 0;
-    // Used to optionally separate methods from each other
-    // with an always marked token
-    int SEPARATOR_TOKEN = 1;
+    final static int C_BLOCK_BEGIN = 2;
+    final static int C_BLOCK_END = 3;
+    final static int C_SCOPE = 4;
+    final static int C_QUESTIONMARK = 5;
+    final static int C_ELLIPSIS = 6;
+    final static int C_ASSIGN = 7;
+    final static int C_DOT = 8;
+    final static int C_ARROW = 9;
+    final static int C_ARROWSTAR = 10;
+    final static int C_AUTO = 11;
+    final static int C_BREAK = 12;
+    final static int C_CASE = 13;
+    final static int C_CATCH = 14;
+    final static int C_CHAR = 15;
+    final static int C_CONST = 16;
+    final static int C_CONTINUE = 17;
+    final static int C_DEFAULT = 18;
+    final static int C_DELETE = 19;
+    final static int C_DO = 20;
+    final static int C_DOUBLE = 21;
+    final static int C_ELSE = 22;
+    final static int C_ENUM = 23;
+    final static int C_EXTERN = 24;
+    final static int C_FLOAT = 25;
+    final static int C_FOR = 26;
+    final static int C_FRIEND = 27;
+    final static int C_GOTO = 28;
+    final static int C_IF = 29;
+    final static int C_INLINE = 30;
+    final static int C_INT = 31;
+    final static int C_LONG = 32;
+    final static int C_NEW = 33;
+    final static int C_PRIVATE = 34;
+    final static int C_PROTECTED = 35;
+    final static int C_PUBLIC = 36;
+    final static int C_REDECLARED = 37;
+    final static int C_REGISTER = 38;
+    final static int C_RETURN = 39;
+    final static int C_SHORT = 40;
+    final static int C_SIGNED = 41;
+    final static int C_SIZEOF = 42;
+    final static int C_STATIC = 43;
+    final static int C_STRUCT = 44;
+    final static int C_CLASS = 45;
+    final static int C_SWITCH = 46;
+    final static int C_TEMPLATE = 47;
+    final static int C_THIS = 48;
+    final static int C_TRY = 49;
+    final static int C_TYPEDEF = 50;
+    final static int C_UNION = 51;
+    final static int C_UNSIGNED = 52;
+    final static int C_VIRTUAL = 53;
+    final static int C_VOID = 54;
+    final static int C_VOLANTILE = 55;
+    final static int C_WHILE = 56;
+    final static int C_OPERATOR = 57;
+    final static int C_THROW = 58;
+    final static int C_ID = 59;
+    final static int C_FUN = 60;
+    final static int C_DOTSTAR = 61;
+    final static int C_NULL = 62;
 
-    int C_BLOCK_BEGIN = 2;
-    int C_BLOCK_END = 3;
-    int C_SCOPE = 4;
-    int C_QUESTIONMARK = 5;
-    int C_ELLIPSIS = 6;
-    int C_ASSIGN = 7;
-    int C_DOT = 8;
-    int C_ARROW = 9;
-    int C_ARROWSTAR = 10;
-    int C_AUTO = 11;
-    int C_BREAK = 12;
-    int C_CASE = 13;
-    int C_CATCH = 14;
-    int C_CHAR = 15;
-    int C_CONST = 16;
-    int C_CONTINUE = 17;
-    int C_DEFAULT = 18;
-    int C_DELETE = 19;
-    int C_DO = 20;
-    int C_DOUBLE = 21;
-    int C_ELSE = 22;
-    int C_ENUM = 23;
-    int C_EXTERN = 24;
-    int C_FLOAT = 25;
-    int C_FOR = 26;
-    int C_FRIEND = 27;
-    int C_GOTO = 28;
-    int C_IF = 29;
-    int C_INLINE = 30;
-    int C_INT = 31;
-    int C_LONG = 32;
-    int C_NEW = 33;
-    int C_PRIVATE = 34;
-    int C_PROTECTED = 35;
-    int C_PUBLIC = 36;
-    int C_REDECLARED = 37;
-    int C_REGISTER = 38;
-    int C_RETURN = 39;
-    int C_SHORT = 40;
-    int C_SIGNED = 41;
-    int C_SIZEOF = 42;
-    int C_STATIC = 43;
-    int C_STRUCT = 44;
-    int C_CLASS = 45;
-    int C_SWITCH = 46;
-    int C_TEMPLATE = 47;
-    int C_THIS = 48;
-    int C_TRY = 49;
-    int C_TYPEDEF = 50;
-    int C_UNION = 51;
-    int C_UNSIGNED = 52;
-    int C_VIRTUAL = 53;
-    int C_VOID = 54;
-    int C_VOLANTILE = 55;
-    int C_WHILE = 56;
-    int C_OPERATOR = 57;
-    int C_THROW = 58;
-    int C_ID = 59;
-    int C_FUN = 60;
-    int C_DOTSTAR = 61;
-    int C_NULL = 62;
-
-    int NUM_DIFF_TOKENS = 63;
+    final static int NUM_DIFF_TOKENS = 63;
 }

jplag.frontend.cpp/src/main/java/de/jplag/cpp/Language.java

@@ -5,16 +5,12 @@ import java.io.File;
 import de.jplag.ErrorConsumer;
 import de.jplag.TokenList;
 
-/*
- * Leider werden C/C++ nicht geparst, sondern nur gescannt...
- */
 public class Language implements de.jplag.Language {
-    private final Scanner scanner;
+    private Scanner scanner; // cpp code is scanned not parsed
 
     public Language(ErrorConsumer program) {
         this.scanner = new Scanner();
         this.scanner.setProgram(program);
-
     }
 
     @Override
@@ -71,9 +67,4 @@ public class Language implements de.jplag.Language {
     public int numberOfTokens() {
         return CPPTokenConstants.NUM_DIFF_TOKENS;
     }
-
-    @Override
-    public String type2string(int type) {
-        return CPPToken.type2string(type);
-    }
 }

jplag.frontend.cpp/src/main/java/de/jplag/cpp/Scanner.java

@@ -5,27 +5,29 @@ import java.io.File;
 import de.jplag.AbstractParser;
 import de.jplag.TokenList;
 
-public class Scanner extends AbstractParser implements CPPTokenConstants {
-    private String actFile;
+public class Scanner extends AbstractParser {
+    private String currentFile;
 
-    private TokenList struct;
+    private TokenList tokens;
 
-    public TokenList scan(File dir, String[] files) {
-        struct = new TokenList();
+    public TokenList scan(File directory, String files[]) {
+        tokens = new TokenList();
         errors = 0;
         CPPScanner scanner = null;// will be initialized in Method scanFile
         for (int i = 0; i < files.length; i++) {
-            actFile = files[i];
+            currentFile = files[i];
             getErrorConsumer().print(null, "Scanning file " + files[i]);
-            if (!CPPScanner.scanFile(dir, files[i], scanner, this))
+            if (!CPPScanner.scanFile(directory, files[i], scanner, this)) {
                 errors++;
-            struct.addToken(new CPPToken(FILE_END, actFile, 1));
+            }
+            tokens.addToken(new CPPToken(CPPTokenConstants.FILE_END, currentFile));
         }
         this.parseEnd();
-        return struct;
+        return tokens;
     }
 
     public void add(int type, Token token) {
-        struct.addToken(new CPPToken(type, actFile, token.beginLine));
+        int length = token.endColumn - token.beginColumn + 1;
+        tokens.addToken(new CPPToken(type, currentFile, token.beginLine, token.beginColumn, length));
     }
 }

jplag.frontend.cpp/src/main/javacc/CPP.jj

@@ -26,7 +26,7 @@ import de.jplag.cpp.NewlineStream;
 public class CPPScanner implements CPPTokenConstants {
     private static Scanner scanner2;
 
-    public static boolean scanFile(File dir, String fileName, de.jplag.cpp.CPPScanner scanner, de.jplag.cpp.Scanner scannerX) {
+    public static boolean scanFile(File dir, String fileName, CPPScanner scanner, Scanner scannerX) {
         try {
             InputStream in = new NewlineStream(new FileInputStream(new File(dir, fileName)));
             if (scanner == null) {

jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/CSharpToken.java

@@ -1,52 +1,17 @@
 package de.jplag.csharp;
 
-import java.io.Serial;
-
 import de.jplag.Token;
 
 public class CSharpToken extends Token implements CSharpTokenConstants {
-    @Serial
-    private static final long serialVersionUID = -169669119665425164L;
-    private int line, column, length;
 
     public CSharpToken(int type, String file, int line, int column, int length) {
         super(type, file, line, column, length);
     }
 
-    @Override
-    public int getLine() {
-        return line;
-    }
-
-    @Override
-    public int getColumn() {
-        return column;
-    }
-
-    @Override
-    public int getLength() {
-        return length;
-    }
-
-    @Override
-    public void setLine(int line) {
-        this.line = line;
-    }
-
-    @Override
-    public void setColumn(int column) {
-        this.column = column;
-    }
-
-    @Override
-    public void setLength(int length) {
-        this.length = length;
-    }
-
-    public static String type2string(int type) {
+    protected String type2string() {
         switch (type) {
             case CSharpTokenConstants.FILE_END:
-                return "**********";
+                return "EOF";
             case _INVOCATION:
                 return "INVOCATION";
             case _OBJECT_CREATION:

jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/CSharpTokenConstants.java

@@ -3,64 +3,58 @@ package de.jplag.csharp;
 import de.jplag.TokenConstants;
 
 public interface CSharpTokenConstants extends TokenConstants {
-    int FILE_END = 0;
+    final static int _INVOCATION = 2;
+    final static int _OBJECT_CREATION = 3;
+    final static int _ARRAY_CREATION = 4;
+    final static int _ASSIGNMENT = 5;
+    final static int _L_BRACE = 6;
+    final static int _R_BRACE = 7;
+    final static int _DECLARE_VAR = 8;
+    final static int _DECLARE_CONST = 9;
+    final static int _IF = 10;
+    final static int _ELSE = 11;
+    final static int _SWITCH_BEGIN = 12;
+    final static int _SWITCH_END = 13;
+    final static int _CASE = 14;
+    final static int _DO = 15;
+    final static int _WHILE = 16;
+    final static int _FOR = 17;
+    final static int _FOREACH = 18;
+    final static int _BREAK = 19;
+    final static int _CONTINUE = 20;
+    final static int _GOTO = 21;
+    final static int _RETURN = 22;
+    final static int _THROW = 23;
+    final static int _CHECKED = 24;
+    final static int _UNCHECKED = 25;
+    final static int _LOCK = 26;
+    final static int _USING = 27;
+    final static int _TRY = 28;
+    final static int _CATCH = 29;
+    final static int _FINALLY = 30;
+    final static int _NAMESPACE_BEGIN = 31;
+    final static int _NAMESPACE_END = 32;
+    final static int _USING_DIRECTIVE = 33;
+    final static int _CLASS_BEGIN = 34;
+    final static int _CLASS_END = 35;
+    final static int _METHOD = 36;
+    final static int _PROPERTY = 37;
+    final static int _EVENT = 38;
+    final static int _INDEXER = 39;
+    final static int _OPERATOR = 40;
+    final static int _CONSTRUCTOR = 41;
+    final static int _STATIC_CONSTR = 42;
+    final static int _DESTRUCTOR = 43;
+    final static int _STRUCT_BEGIN = 44;
+    final static int _STRUCT_END = 45;
+    final static int _INTERFACE_BEGIN = 46;
+    final static int _INTERFACE_END = 47;
+    final static int _ENUM = 48;
+    final static int _DELEGATE = 49;
+    final static int _ATTRIBUTE = 50;
+    final static int _END_IF = 51;
+    final static int _UNSAFE = 52;
+    final static int _FIXED = 53;
 
-    // Used to optionally separate methods from each other
-    // with an always marked token
-    int SEPARATOR_TOKEN = 1;
-
-    int _INVOCATION = 2;
-    int _OBJECT_CREATION = 3;
-    int _ARRAY_CREATION = 4;
-    int _ASSIGNMENT = 5;
-    int _L_BRACE = 6;
-    int _R_BRACE = 7;
-    int _DECLARE_VAR = 8;
-    int _DECLARE_CONST = 9;
-    int _IF = 10;
-    int _ELSE = 11;
-    int _SWITCH_BEGIN = 12;
-    int _SWITCH_END = 13;
-    int _CASE = 14;
-    int _DO = 15;
-    int _WHILE = 16;
-    int _FOR = 17;
-    int _FOREACH = 18;
-    int _BREAK = 19;
-    int _CONTINUE = 20;
-    int _GOTO = 21;
-    int _RETURN = 22;
-    int _THROW = 23;
-    int _CHECKED = 24;
-    int _UNCHECKED = 25;
-    int _LOCK = 26;
-    int _USING = 27;
-    int _TRY = 28;
-    int _CATCH = 29;
-    int _FINALLY = 30;
-    int _NAMESPACE_BEGIN = 31;
-    int _NAMESPACE_END = 32;
-    int _USING_DIRECTIVE = 33;
-    int _CLASS_BEGIN = 34;
-    int _CLASS_END = 35;
-    int _METHOD = 36;
-    int _PROPERTY = 37;
-    int _EVENT = 38;
-    int _INDEXER = 39;
-    int _OPERATOR = 40;
-    int _CONSTRUCTOR = 41;
-    int _STATIC_CONSTR = 42;
-    int _DESTRUCTOR = 43;
-    int _STRUCT_BEGIN = 44;
-    int _STRUCT_END = 45;
-    int _INTERFACE_BEGIN = 46;
-    int _INTERFACE_END = 47;
-    int _ENUM = 48;
-    int _DELEGATE = 49;
-    int _ATTRIBUTE = 50;
-    int _END_IF = 51;
-    int _UNSAFE = 52;
-    int _FIXED = 53;
-
-    int NUM_DIFF_TOKENS = 54;
+    final static int NUM_DIFF_TOKENS = 54;
 }

jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/Language.java

@@ -26,7 +26,7 @@ public class Language implements de.jplag.Language {
 
     @Override
     public String getName() {
-        return "C# 1.2 AbstractParser";
+        return "C# 1.2 Parser";
     }
 
     @Override
@@ -68,9 +68,4 @@ public class Language implements de.jplag.Language {
     public int numberOfTokens() {
         return CSharpTokenConstants.NUM_DIFF_TOKENS;
     }
-
-    @Override
-    public String type2string(int type) {
-        return CSharpToken.type2string(type);
-    }
 }

jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/Parser.java

@@ -11,28 +11,28 @@ import de.jplag.TokenList;
 import de.jplag.csharp.grammar.CSharpLexer;
 import de.jplag.csharp.grammar.CSharpParser;
 
-public class Parser extends AbstractParser implements CSharpTokenConstants {
-    private TokenList struct;
+public class Parser extends AbstractParser {
+    private TokenList tokens;
     private String currentFile;
 
-    public TokenList parse(File dir, String[] files) {
-        struct = new TokenList();
+    public TokenList parse(File directory, String files[]) {
+        tokens = new TokenList();
         errors = 0;
         for (int i = 0; i < files.length; i++) {
-            if (!parseFile(dir, files[i]))
+            if (!parseFile(directory, files[i]))
                 errors++;
-            struct.addToken(new CSharpToken(FILE_END, files[i], -1, -1, -1));
+            tokens.addToken(new CSharpToken(CSharpTokenConstants.FILE_END, files[i], -1, -1, -1));
         }
         this.parseEnd();
-        return struct;
+        return tokens;
     }
 
     private boolean parseFile(File dir, String file) {
         try {
-            FileInputStream fis = new FileInputStream(new File(dir, file));
+            FileInputStream input = new FileInputStream(new File(dir, file));
             currentFile = file;
             // Create a scanner that reads from the input stream passed to us
-            CSharpLexer lexer = new CSharpLexer(new UnicodeReader(fis, StandardCharsets.UTF_8));
+            CSharpLexer lexer = new CSharpLexer(new UnicodeReader(input, StandardCharsets.UTF_8));
             lexer.setFilename(file);
             lexer.setTabSize(1);
 
@@ -44,7 +44,7 @@ public class Parser extends AbstractParser implements CSharpTokenConstants {
             parser.compilation_unit();
 
             // close file
-            fis.close();
+            input.close();
         } catch (Exception e) {
             getErrorConsumer().addError("  Parsing Error in '" + file + "':\n  " + e.toString());
             return false;
@@ -52,15 +52,15 @@ public class Parser extends AbstractParser implements CSharpTokenConstants {
         return true;
     }
 
-    private void add(int type, Token tok) {
-        if (tok == null) {
+    private void add(int type, Token token) {
+        if (token == null) {
             System.out.println("tok == null  ERROR!");
             return;
         }
-        struct.addToken(new CSharpToken(type, currentFile, tok.getLine(), tok.getColumn(), tok.getText().length()));
+        tokens.addToken(new CSharpToken(type, currentFile, token.getLine(), token.getColumn(), token.getText().length()));
     }
 
-    public void add(int type, CSharpParser p) {
-        add(type, p.getLastConsumedToken());
+    public void add(int type, CSharpParser parser) {
+        add(type, parser.getLastConsumedToken());
     }
 }

jplag.frontend.java/src/main/java/de/jplag/java/JavaToken.java

@@ -1,187 +1,153 @@
 package de.jplag.java;
 
-import java.io.Serial;
-
 import de.jplag.Token;
 
 public class JavaToken extends Token implements JavaTokenConstants {
-    @Serial
-    private static final long serialVersionUID = -383581430479870696L;
-    private int line, column, length;
-
-    public JavaToken(int type, String file, int line, int col, int length) {
-        super(type, file, line, col, length);
-    }
-
-    @Override
-    public int getLine() {
-        return line;
-    }
-
-    @Override
-    public int getColumn() {
-        return column;
-    }
 
-    @Override
-    public int getLength() {
-        return length;
+    public JavaToken(int type, String file, int line, int column, int length) {
+        super(type, file, line, column, length);
     }
 
     @Override
-    public void setLine(int line) {
-        this.line = line;
-    }
-
-    @Override
-    public void setColumn(int column) {
-        this.column = column;
-    }
-
-    @Override
-    public void setLength(int length) {
-        this.length = length;
-    }
-
-    public static String type2string(int type) {
+    protected String type2string() {
         switch (type) {
             case JavaTokenConstants.FILE_END:
-                return "********";
+                return "EOF";
             case JavaTokenConstants.SEPARATOR_TOKEN:
                 return "METHOD_SEPARATOR";
             case J_PACKAGE:
-                return "PACKAGE ";
+                return "PACKAGE";
             case J_IMPORT:
-                return "IMPORT  ";
+                return "IMPORT";
             case J_CLASS_BEGIN:
-                return "CLASS{  ";
+                return "CLASS{";
             case J_CLASS_END:
-                return "}CLASS  ";
+                return "}CLASS";
             case J_METHOD_BEGIN:
-                return "METHOD{ ";
+                return "METHOD{";
             case J_METHOD_END:
-                return "}METHOD ";
+                return "}METHOD";
             case J_VARDEF:
-                return "VARDEF  ";
+                return "VARDEF";
             case J_SYNC_BEGIN:
-                return "SYNC{   ";
+                return "SYNC{";
             case J_SYNC_END:
-                return "}SYNC   ";
+                return "}SYNC";
             case J_DO_BEGIN:
-                return "DO{     ";
+                return "DO{";
             case J_DO_END:
-                return "}DO     ";
+                return "}DO";
             case J_WHILE_BEGIN:
-                return "WHILE{  ";
+                return "WHILE{";
             case J_WHILE_END:
-                return "}WHILE  ";
+                return "}WHILE";
             case J_FOR_BEGIN:
-                return "FOR{    ";
+                return "FOR{";
             case J_FOR_END:
-                return "}FOR    ";
+                return "}FOR";
             case J_SWITCH_BEGIN:
-                return "SWITCH{ ";
+                return "SWITCH{";
             case J_SWITCH_END:
-                return "}SWITCH ";
+                return "}SWITCH";
             case J_CASE:
-                return "CASE    ";
+                return "CASE";
             case J_TRY_BEGIN:
-                return "TRY{    ";
+                return "TRY{";
             case J_CATCH_BEGIN:
-                return "CATCH{  ";
+                return "CATCH{";
             case J_CATCH_END:
-                return "}CATCH  ";
+                return "}CATCH";
             case J_FINALLY:
-                return "FINALLY ";
+                return "FINALLY";
             case J_IF_BEGIN:
-                return "IF{     ";
+                return "IF{";
             case J_ELSE:
-                return "ELSE    ";
+                return "ELSE";
             case J_IF_END:
-                return "}IF     ";
+                return "}IF";
             case J_COND:
-                return "COND    ";
+                return "COND";
             case J_BREAK:
-                return "BREAK   ";
+                return "BREAK";
             case J_CONTINUE:
                 return "CONTINUE";
             case J_RETURN:
-                return "RETURN  ";
+                return "RETURN";
             case J_THROW:
-                return "THROW   ";
+                return "THROW";
             case J_IN_CLASS_BEGIN:
                 return "INCLASS{";
             case J_IN_CLASS_END:
                 return "}INCLASS";
             case J_APPLY:
-                return "APPLY   ";
+                return "APPLY";
             case J_NEWCLASS:
                 return "NEWCLASS";
             case J_NEWARRAY:
                 return "NEWARRAY";
             case J_ASSIGN:
-                return "ASSIGN  ";
+                return "ASSIGN";
             case J_INTERFACE_BEGIN:
-                return "INTERF{ ";
+                return "INTERF{";
             case J_INTERFACE_END:
-                return "}INTERF ";
+                return "}INTERF";
             case J_CONSTR_BEGIN:
-                return "CONSTR{ ";
+                return "CONSTR{";
             case J_CONSTR_END:
-                return "}CONSTR ";
+                return "}CONSTR";
             case J_INIT_BEGIN:
-                return "INIT{   ";
+                return "INIT{";
             case J_INIT_END:
-                return "}INIT   ";
+                return "}INIT";
             case J_VOID:
-                return "VOID    ";
+                return "VOID";
             case J_ARRAY_INIT_BEGIN:
                 return "ARRINIT{";
             case J_ARRAY_INIT_END:
                 return "ARRINIT}";
             case J_ENUM_BEGIN:
-                return "ENUM{   ";
+                return "ENUM{";
             case J_ENUM_CLASS_BEGIN:
                 return "ENUM_CLA";
             case J_ENUM_END:
-                return "}ENUM   ";
+                return "}ENUM";
             case J_GENERIC:
-                return "GENERIC ";
+                return "GENERIC";
             case J_ASSERT:
-                return "ASSERT  ";
+                return "ASSERT";
             case J_ANNO:
-                return "ANNO    ";
+                return "ANNO";
             case J_ANNO_MARKER:
                 return "ANNOMARK";
             case J_ANNO_M_BEGIN:
-                return "ANNO_M{ ";
+                return "ANNO_M{";
             case J_ANNO_M_END:
-                return "}ANNO_M ";
+                return "}ANNO_M";
             case J_ANNO_T_BEGIN:
-                return "ANNO_T{ ";
+                return "ANNO_T{";
             case J_ANNO_T_END:
-                return "}ANNO_T ";
+                return "}ANNO_T";
             case J_ANNO_C_BEGIN:
-                return "ANNO_C{ ";
+                return "ANNO_C{";
             case J_ANNO_C_END:
-                return "}ANNO_C ";
+                return "}ANNO_C";
             case J_MODULE_BEGIN:
-                return "MODULE{ ";
+                return "MODULE{";
             case J_MODULE_END:
-                return "}MODULE ";
+                return "}MODULE";
             case J_EXPORTS:
-                return "EXPORTS ";
+                return "EXPORTS";
             case J_PROVIDES:
                 return "PROVIDES";
             case J_REQUIRES:
                 return "REQUIRES";
 
             case J_TRY_WITH_RESOURCE:
-                return "TRY_RES ";
+                return "TRY_RES";
 
             default:
-                System.err.println("*UNKNOWN: " + type);
-                return "*UNKNOWN" + type;
+                System.err.println("UNKNOWN: " + type);
+                return "<UNKNOWN" + type + ">";
         }
     }
 }

jplag.frontend.java/src/main/java/de/jplag/java/JavaTokenConstants.java

@@ -3,82 +3,79 @@ package de.jplag.java;
 import de.jplag.TokenConstants;
 
 public interface JavaTokenConstants extends TokenConstants {
-    int FILE_END = 0;
 
-    // check used to mark tokens that are actually generated by the adapter
-    int SEPARATOR_TOKEN = 1;// @formatter:off
-
-    int J_PACKAGE = 2;                // check
-    int J_IMPORT = 3;                // check
-    int J_CLASS_BEGIN = 4;            // check
-    int J_CLASS_END = 5;            // check
-    int J_METHOD_BEGIN = 6;        // check
-    int J_METHOD_END = 7;            // check
-    int J_VARDEF = 8;                // check
-    int J_SYNC_BEGIN = 9;            // check
-    int J_SYNC_END = 10;            // check
-    int J_DO_BEGIN = 11;            // check
-    int J_DO_END = 12;                // check
-    int J_WHILE_BEGIN = 13;        // check
-    int J_WHILE_END = 14;            // check
-    int J_FOR_BEGIN = 15;            // check
-    int J_FOR_END = 16;            // check
-    int J_SWITCH_BEGIN = 17;        // check
-    int J_SWITCH_END = 18;            // check
-    int J_CASE = 19;                // check
-    int J_TRY_BEGIN = 20;            // check
-    int J_CATCH_BEGIN = 21;        // check
-    int J_CATCH_END = 22;        // check
-    int J_FINALLY = 23;            // check
-    int J_IF_BEGIN = 24;            // check
-    int J_ELSE = 25;                // check
-    int J_IF_END = 26;                // check
-    int J_COND = 27;                // check
-    int J_BREAK = 28;                // check
-    int J_CONTINUE = 29;            // check
-    int J_RETURN = 30;                // check
-    int J_THROW = 31;                // check
-    int J_IN_CLASS_BEGIN = 32;        //
-    int J_IN_CLASS_END = 33;        //
-    int J_APPLY = 34;                // check
-    int J_NEWCLASS = 35;            // check
-    int J_NEWARRAY = 36;            // check
-    int J_ASSIGN = 37;                // check
-    int J_INTERFACE_BEGIN = 38;    // check
-    int J_INTERFACE_END = 39;        // check
-    int J_CONSTR_BEGIN = 40;        //
-    int J_CONSTR_END = 41;            //
-    int J_INIT_BEGIN = 42;            // check
-    int J_INIT_END = 43;            // check
-    int J_VOID = 44;                //
-    int J_ARRAY_INIT_BEGIN = 45;    // check
-    int J_ARRAY_INIT_END = 46;    // check
+    // @formatter:off
+    final static int J_PACKAGE = 2;				// check
+    final static int J_IMPORT = 3;				// check
+    final static int J_CLASS_BEGIN = 4;			// check
+    final static int J_CLASS_END = 5;			// check
+    final static int J_METHOD_BEGIN = 6;		// check
+    final static int J_METHOD_END = 7;			// check
+    final static int J_VARDEF = 8;				// check
+    final static int J_SYNC_BEGIN = 9;			// check
+    final static int J_SYNC_END = 10;			// check
+    final static int J_DO_BEGIN = 11;			// check
+    final static int J_DO_END = 12;				// check
+    final static int J_WHILE_BEGIN = 13;		// check
+    final static int J_WHILE_END = 14;			// check
+    final static int J_FOR_BEGIN = 15;			// check
+    final static int J_FOR_END = 16;			// check
+    final static int J_SWITCH_BEGIN = 17;		// check
+    final static int J_SWITCH_END = 18;			// check
+    final static int J_CASE = 19;				// check
+    final static int J_TRY_BEGIN = 20;			// check
+    final static int J_CATCH_BEGIN = 21;    	// check
+    final static int J_CATCH_END = 22;      	// check
+    final static int J_FINALLY = 23;			// check
+    final static int J_IF_BEGIN = 24;			// check
+    final static int J_ELSE = 25;				// check
+    final static int J_IF_END = 26;				// check
+    final static int J_COND = 27;				// check
+    final static int J_BREAK = 28;				// check
+    final static int J_CONTINUE = 29;			// check
+    final static int J_RETURN = 30;				// check
+    final static int J_THROW = 31;				// check
+    final static int J_IN_CLASS_BEGIN = 32;		//
+    final static int J_IN_CLASS_END = 33;		//
+    final static int J_APPLY = 34;				// check
+    final static int J_NEWCLASS = 35;			// check
+    final static int J_NEWARRAY = 36;			// check
+    final static int J_ASSIGN = 37;				// check
+    final static int J_INTERFACE_BEGIN = 38;	// check
+    final static int J_INTERFACE_END = 39;		// check
+    final static int J_CONSTR_BEGIN = 40;		//
+    final static int J_CONSTR_END = 41;			//
+    final static int J_INIT_BEGIN = 42;			// check
+    final static int J_INIT_END = 43;			// check
+    final static int J_VOID = 44;				//
+    final static int J_ARRAY_INIT_BEGIN = 45;	// check
+    final static int J_ARRAY_INIT_END = 46; 	// check
 
     // new in 1.5:
-    int J_ENUM_BEGIN = 47;            // check
-    int J_ENUM_CLASS_BEGIN = 48;    // ?? doesn't exist in JAVAC
-    int J_ENUM_END = 49;            // check
-    int J_GENERIC = 50;            // check
-    int J_ASSERT = 51;                // check
+    final static int J_ENUM_BEGIN = 47;			// check
+    final static int J_ENUM_CLASS_BEGIN = 48;	// ?? doesn't exist in JAVAC
+    final static int J_ENUM_END = 49;			// check
+    final static int J_GENERIC = 50;			// check
+    final static int J_ASSERT = 51;				// check
 
-    int J_ANNO = 52;                // check
-    int J_ANNO_MARKER = 53;        // ??
-    int J_ANNO_M_BEGIN = 54;        // ??
-    int J_ANNO_M_END = 55;            // ??
-    int J_ANNO_T_BEGIN = 56;        // check
-    int J_ANNO_T_END = 57;            // check
-    int J_ANNO_C_BEGIN = 58;        // ??
-    int J_ANNO_C_END = 59;            // ??
+    final static int J_ANNO = 52;				// check
+    final static int J_ANNO_MARKER = 53;		// ??
+    final static int J_ANNO_M_BEGIN = 54;		// ??
+    final static int J_ANNO_M_END = 55;			// ??
+    final static int J_ANNO_T_BEGIN = 56;		// check
+    final static int J_ANNO_T_END = 57;			// check
+    final static int J_ANNO_C_BEGIN = 58;		// ??
+    final static int J_ANNO_C_END = 59;			// ??
 
     // new in 1.7
-    int J_TRY_WITH_RESOURCE = 60;    // check
+    final static int J_TRY_WITH_RESOURCE = 60; 	// check
 
     // new in 1.9
-    int J_REQUIRES = 61;            // check
-    int J_PROVIDES = 62;            // check
-    int J_EXPORTS = 63;            // check
-    int J_MODULE_BEGIN = 64;       // check
-    int J_MODULE_END = 65;        // check
+    final static int J_REQUIRES = 61; 	        // check
+    final static int J_PROVIDES = 62; 	        // check
+    final static int J_EXPORTS = 63; 	        // check
+    final static int J_MODULE_BEGIN = 64;       // check
+    final static int J_MODULE_END = 65; 	    // check
 
-    int NUM_DIFF_TOKENS = 66;        // @formatter:on
+    final static int NUM_DIFF_TOKENS = 66;	    // @formatter:on
 }

jplag.frontend.java/src/main/java/de/jplag/java/JavacAdapter.java

@@ -23,10 +23,11 @@ public class JavacAdapter {
 
     private static final JavaCompiler javac = ToolProvider.getSystemJavaCompiler();
 
-    public int parseFiles(File dir, File[] pathedFiles, final Parser parser) {
-        final StandardJavaFileManager jfm = javac.getStandardFileManager(null, null, StandardCharsets.UTF_8);
+    public int parseFiles(File directory, File[] pathedFiles, final Parser parser) {
+        final StandardJavaFileManager fileManager = javac.getStandardFileManager(null, null, StandardCharsets.UTF_8);
         DiagnosticCollector<? super JavaFileObject> diagListen = new DiagnosticCollector<>();
-        final JavaCompiler.CompilationTask task = javac.getTask(null, jfm, diagListen, null, null, jfm.getJavaFileObjects(pathedFiles));
+        final JavaCompiler.CompilationTask task = javac.getTask(null, fileManager, diagListen, null, null,
+                fileManager.getJavaFileObjects(pathedFiles));
         Iterable<? extends CompilationUnitTree> asts = Collections.emptyList();
         try {
             asts = ((JavacTask) task).parse();
@@ -37,10 +38,10 @@ public class JavacAdapter {
         final SourcePositions positions = trees.getSourcePositions();
         for (final CompilationUnitTree ast : asts) {
             final String filename;
-            if (dir == null)
+            if (directory == null)
                 filename = ast.getSourceFile().getName();
             else {
-                filename = Paths.get(dir.toURI()).relativize(Paths.get(ast.getSourceFile().toUri())).toString();
+                filename = Paths.get(directory.toURI()).relativize(Paths.get(ast.getSourceFile().toUri())).toString();
             }
             final LineMap map = ast.getLineMap();
             ast.accept(new TokenGeneratingTreeScanner(filename, parser, map, positions, ast), null);

jplag.frontend.java/src/main/java/de/jplag/java/Language.java

@@ -56,11 +56,6 @@ public class Language implements de.jplag.Language {
         return JavaTokenConstants.NUM_DIFF_TOKENS;
     }
 
-    @Override
-    public String type2string(int type) {
-        return JavaToken.type2string(type);
-    }
-
     @Override
     public TokenList parse(File dir, String[] files) {
         return this.parser.parse(dir, files);

jplag.frontend.java/src/main/java/de/jplag/java/Parser.java

@@ -6,23 +6,23 @@ import de.jplag.AbstractParser;
 import de.jplag.TokenList;
 
 public class Parser extends AbstractParser {
-    private TokenList tokenList;
+    private TokenList tokens;
 
-    public TokenList parse(File dir, String[] files) {
-        tokenList = new TokenList();
+    public TokenList parse(File directory, String files[]) {
+        tokens = new TokenList();
         errors = 0;
-        File[] pathedFiles = new File[files.length];
+        File pathedFiles[] = new File[files.length];
         for (int i = 0; i < files.length; i++) {
-            pathedFiles[i] = new File(dir, files[i]);
+            pathedFiles[i] = new File(directory, files[i]);
         }
         JavacAdapter javac = new JavacAdapter();
-        errors += javac.parseFiles(dir, pathedFiles, this);
+        errors += javac.parseFiles(directory, pathedFiles, this);
         this.parseEnd();
-        return tokenList;
+        return tokens;
     }
 
     public void add(int type, String filename, long line, long col, long length) {
-        tokenList.addToken(new JavaToken(type, filename, (int) line, (int) col, (int) length));
+        tokens.addToken(new JavaToken(type, filename, (int) line, (int) col, (int) length));
     }
 
     public void errorsInc() {

jplag.frontend.java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java

@@ -28,6 +28,7 @@ import com.sun.source.tree.PackageTree;
 import com.sun.source.tree.ProvidesTree;
 import com.sun.source.tree.RequiresTree;
 import com.sun.source.tree.ReturnTree;
+import com.sun.source.tree.SwitchExpressionTree;
 import com.sun.source.tree.SwitchTree;
 import com.sun.source.tree.SynchronizedTree;
 import com.sun.source.tree.ThrowTree;
@@ -182,6 +183,16 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
         return ret;
     }
 
+    @Override
+    public Object visitSwitchExpression(SwitchExpressionTree node, Object parameterValue) {
+        long start = positions.getStartPosition(ast, node);
+        long end = positions.getEndPosition(ast, node);
+        parser.add(JavaTokenConstants.J_SWITCH_BEGIN, filename, map.getLineNumber(start), map.getColumnNumber(start), 6);
+        Object result = super.visitSwitchExpression(node, parameterValue);
+        parser.add(JavaTokenConstants.J_SWITCH_END, filename, map.getLineNumber(end - 1), map.getColumnNumber(end - 1), 1);
+        return result;
+    }
+
     @Override
     public Object visitCase(CaseTree node, Object p) {
         long n = positions.getStartPosition(ast, node);

jplag.frontend.python-3/src/main/java/de/jplag/python3/Language.java

@@ -26,7 +26,7 @@ public class Language implements de.jplag.Language {
 
     @Override
     public String getName() {
-        return "Python3 AbstractParser";
+        return "Python3 Parser";
     }
 
     @Override
@@ -68,9 +68,4 @@ public class Language implements de.jplag.Language {
     public int numberOfTokens() {
         return Python3TokenConstants.NUM_DIFF_TOKENS;
     }
-
-    @Override
-    public String type2string(int type) {
-        return Python3Token.type2string(type);
-    }
 }

jplag.frontend.python-3/src/main/java/de/jplag/python3/Parser.java

@@ -18,34 +18,34 @@ import de.jplag.python3.grammar.Python3Lexer;
 import de.jplag.python3.grammar.Python3Parser;
 import de.jplag.python3.grammar.Python3Parser.File_inputContext;
 
-public class Parser extends AbstractParser implements Python3TokenConstants {
+public class Parser extends AbstractParser {
 
-    private TokenList struct = new TokenList();
+    private TokenList tokens = new TokenList();
     private String currentFile;
 
-    public TokenList parse(File dir, String[] files) {
-        struct = new TokenList();
+    public TokenList parse(File directory, String files[]) {
+        tokens = new TokenList();
         errors = 0;
         for (int i = 0; i < files.length; i++) {
             getErrorConsumer().print(null, "Parsing file " + files[i]);
-            if (!parseFile(dir, files[i])) {
+            if (!parseFile(directory, files[i])) {
                 errors++;
             }
             System.gc();// Emeric
-            struct.addToken(new Python3Token(FILE_END, files[i], -1, -1, -1));
+            tokens.addToken(new Python3Token(Python3TokenConstants.FILE_END, files[i], -1, -1, -1));
         }
         this.parseEnd();
-        return struct;
+        return tokens;
     }
 
-    private boolean parseFile(File dir, String file) {
-        BufferedInputStream fis;
+    private boolean parseFile(File directory, String file) {
+        BufferedInputStream inputStream;
 
         CharStream input;
         try {
-            fis = new BufferedInputStream(new FileInputStream(new File(dir, file)));
+            inputStream = new BufferedInputStream(new FileInputStream(new File(directory, file)));
             currentFile = file;
-            input = CharStreams.fromStream(fis);
+            input = CharStreams.fromStream(inputStream);
 
             // create a lexer that feeds off of input CharStream
             Python3Lexer lexer = new Python3Lexer(input);
@@ -71,13 +71,13 @@ public class Parser extends AbstractParser implements Python3TokenConstants {
         return true;
     }
 
-    public void add(int type, Token tok) {
-        struct.addToken(new Python3Token(type, (currentFile == null ? "null" : currentFile), tok.getLine(), tok.getCharPositionInLine() + 1,
-                tok.getText().length()));
+    public void add(int type, Token token) {
+        tokens.addToken(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(), token.getCharPositionInLine() + 1,
+                token.getText().length()));
     }
 
-    public void addEnd(int type, Token tok) {
-        struct.addToken(new Python3Token(type, (currentFile == null ? "null" : currentFile), tok.getLine(),
-                struct.getToken(struct.size() - 1).getColumn() + 1, 0));
+    public void addEnd(int type, Token token) {
+        tokens.addToken(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(),
+                tokens.getToken(tokens.size() - 1).getColumn() + 1, 0));
     }
 }

jplag.frontend.python-3/src/main/java/de/jplag/python3/Python3Token.java

@@ -1,54 +1,17 @@
 package de.jplag.python3;
 
-import java.io.Serial;
-
 import de.jplag.Token;
 
 public class Python3Token extends Token implements Python3TokenConstants {
 
-    @Serial
-    private static final long serialVersionUID = 1485877548175917943L;
-
-    private int line, column, length;
-
     public Python3Token(int type, String file, int line, int column, int length) {
         super(type, file, line, column, length);
     }
 
-    @Override
-    public int getLine() {
-        return line;
-    }
-
-    @Override
-    public int getColumn() {
-        return column;
-    }
-
-    @Override
-    public int getLength() {
-        return length;
-    }
-
-    @Override
-    public void setLine(int line) {
-        this.line = line;
-    }
-
-    @Override
-    public void setColumn(int column) {
-        this.column = column;
-    }
-
-    @Override
-    public void setLength(int length) {
-        this.length = length;
-    }
-
-    public static String type2string(int type) {
+    protected String type2string() {
         switch (type) {
             case Python3TokenConstants.FILE_END:
-                return "********";
+                return "EOF";
             case Python3TokenConstants.SEPARATOR_TOKEN:
                 return "METHOD_SEPARATOR";
 

jplag.frontend.python-3/src/main/java/de/jplag/python3/Python3TokenConstants.java

@@ -3,40 +3,36 @@ package de.jplag.python3;
 import de.jplag.TokenConstants;
 
 public interface Python3TokenConstants extends TokenConstants {
+    final static int IMPORT = 2;
+    final static int CLASS_BEGIN = 3;
+    final static int CLASS_END = 4;
+    final static int METHOD_BEGIN = 5;
+    final static int METHOD_END = 6;
+    final static int ASSIGN = 7;
+    final static int WHILE_BEGIN = 8;
+    final static int WHILE_END = 9;
+    final static int FOR_BEGIN = 10;
+    final static int FOR_END = 11;
+    final static int TRY_BEGIN = 12;
+    final static int EXCEPT_BEGIN = 13;
+    final static int EXCEPT_END = 14;
+    final static int FINALLY = 15;
+    final static int IF_BEGIN = 16;
+    final static int IF_END = 18;
+    final static int APPLY = 19;
+    final static int BREAK = 20;
+    final static int CONTINUE = 21;
+    final static int RETURN = 22;
+    final static int RAISE = 23;
+    final static int DEC_BEGIN = 24;
+    final static int DEC_END = 25;
+    final static int LAMBDA = 26;
+    final static int ARRAY = 27;
+    final static int ASSERT = 28;
+    final static int YIELD = 29;
+    final static int DEL = 30;
+    final static int WITH_BEGIN = 31;
+    final static int WITH_END = 32;
 
-    int FILE_END = 0;
-    int SEPARATOR_TOKEN = 1;
-
-    int IMPORT = 2;
-    int CLASS_BEGIN = 3;
-    int CLASS_END = 4;
-    int METHOD_BEGIN = 5;
-    int METHOD_END = 6;
-    int ASSIGN = 7;
-    int WHILE_BEGIN = 8;
-    int WHILE_END = 9;
-    int FOR_BEGIN = 10;
-    int FOR_END = 11;
-    int TRY_BEGIN = 12;
-    int EXCEPT_BEGIN = 13;
-    int EXCEPT_END = 14;
-    int FINALLY = 15;
-    int IF_BEGIN = 16;
-    int IF_END = 18;
-    int APPLY = 19;
-    int BREAK = 20;
-    int CONTINUE = 21;
-    int RETURN = 22;
-    int RAISE = 23;
-    int DEC_BEGIN = 24;
-    int DEC_END = 25;
-    int LAMBDA = 26;
-    int ARRAY = 27;
-    int ASSERT = 28;
-    int YIELD = 29;
-    int DEL = 30;
-    int WITH_BEGIN = 31;
-    int WITH_END = 32;
-
-    int NUM_DIFF_TOKENS = 33;
+    final static int NUM_DIFF_TOKENS = 33;
 }

jplag.frontend.scheme/src/main/java/de/jplag/scheme/Language.java

@@ -27,7 +27,7 @@ public class Language implements de.jplag.Language {
 
     @Override
     public String getName() {
-        return "SchemeR4RS AbstractParser [basic markup]";
+        return "SchemeR4RS Parser [basic markup]";
     }
 
     @Override
@@ -69,9 +69,4 @@ public class Language implements de.jplag.Language {
     public int numberOfTokens() {
         return SchemeTokenConstants.NUM_DIFF_TOKENS;
     }
-
-    @Override
-    public String type2string(int type) {
-        return SchemeToken.type2string(type);
-    }
 }

jplag.frontend.scheme/src/main/java/de/jplag/scheme/Parser.java

@@ -5,32 +5,28 @@ import java.io.File;
 import de.jplag.AbstractParser;
 import de.jplag.TokenList;
 
-public class Parser extends AbstractParser implements SchemeTokenConstants {
-    private String actFile;
+public class Parser extends AbstractParser {
+    private String currentFile;
 
-    private TokenList struct;
+    private TokenList tokens;
 
-    public TokenList parse(File dir, String[] files) {
-        struct = new TokenList();
+    public TokenList parse(File directory, String files[]) {
+        tokens = new TokenList();
         errors = 0;
-        SchemeParser parser = null;// no worry it will be reinitialized
-        // in method parseFile(...)
         for (int i = 0; i < files.length; i++) {
-            actFile = files[i];
+            currentFile = files[i];
             getErrorConsumer().print(null, "Parsing file " + files[i]);
-            if (!SchemeParser.parseFile(dir, files[i], parser, this))
+            if (!SchemeParser.parseFile(directory, files[i], null, this))
                 errors++;
-            struct.addToken(new SchemeToken(FILE_END, actFile, 1));
+            tokens.addToken(new SchemeToken(SchemeTokenConstants.FILE_END, currentFile));
         }
         this.parseEnd();
-        return struct;
+        return tokens;
     }
 
     public void add(int type, Token token) {
-        struct.addToken(new SchemeToken(type, actFile, token.beginLine));
-        /*
-         * System.out.println(token.beginLine+"\t"+ (new SchemeToken(0,null,0)).type2string(type)+"\t"+ token.image);
-         */
+        int length = token.endColumn - token.beginColumn + 1;
+        tokens.addToken(new SchemeToken(type, currentFile, token.beginLine, token.endLine, length));
     }
 
 }

jplag.frontend.scheme/src/main/java/de/jplag/scheme/SchemeToken.java

@@ -1,42 +1,21 @@
 package de.jplag.scheme;
 
-import java.io.Serial;
-
 import de.jplag.Token;
 
 public class SchemeToken extends Token implements SchemeTokenConstants {
-    @Serial
-    private static final long serialVersionUID = -143418583849687339L;
-    private int line;
-
-    public SchemeToken(int type, String file, int line) {
-        super(type, file, line);
-    }
-
-    @Override
-    public int getLine() {
-        return line;
-    }
-
-    @Override
-    public void setLine(int line) {
-        this.line = line;
-    }
 
-    @Override
-    public int getColumn() {
-        return 0;
+    public SchemeToken(int type, String file, int line, int column, int length) {
+        super(type, file, line, column, length);
     }
 
-    @Override
-    public int getLength() {
-        return 0;
+    public SchemeToken(int type, String file) {
+        super(type, file, 1, 0, 0);
     }
 
-    public static String type2string(int type) {
+    protected String type2string() {
         switch (type) {
             case SchemeTokenConstants.FILE_END:
-                return "********";
+                return "EOF";
             case S_BOOL:
                 return "BOOL    ";
             case S_NUMBER:

jplag.frontend.scheme/src/main/java/de/jplag/scheme/SchemeTokenConstants.java

@@ -3,53 +3,48 @@ package de.jplag.scheme;
 import de.jplag.TokenConstants;
 
 public interface SchemeTokenConstants extends TokenConstants {
-    int FILE_END = 0;
 
-    // Used to optionally separate methods from each other
-    // with an always marked token
-    int SEPARATOR_TOKEN = 1;
+    final static int S_BOOL = 2;
+    final static int S_NUMBER = 3;
+    final static int S_CHAR = 4;
+    final static int S_STRING = 5;
+    final static int S_ID = 6;
+    final static int S_LIST_BEGIN = 7;
+    final static int S_LIST_END = 8;
+    final static int S_VECTOR_BEGIN = 9;
+    final static int S_VECTOR_END = 10;
+    final static int S_LITERAL = 11;
+    final static int S_QUOT_BEGIN = 12;
+    final static int S_QUOT_END = 13;
+    final static int S_CALL = 14;
+    final static int S_LAMBDA_BEGIN = 15;
+    final static int S_LAMBDA_END = 16;
+    final static int S_FORMAL_BEGIN = 17;
+    final static int S_FORMAL_END = 18;
+    final static int S_BODY_BEGIN = 19;
+    final static int S_BODY_END = 20;
+    final static int S_IF_BEGIN = 21;
+    final static int S_IF_END = 22;
+    final static int S_ALTERN = 23;
+    final static int S_ASSIGN_BEGIN = 24;
+    final static int S_ASSIGN_END = 25;
+    final static int S_COND_BEGIN = 26;
+    final static int S_COND_END = 27;
+    final static int S_ELSE = 28;
+    final static int S_CASE_BEGIN = 29;
+    final static int S_CASE_END = 30;
+    final static int S_DO_BEGIN = 31;
+    final static int S_DO_END = 32;
+    final static int S_COMMAND = 33;
+    final static int S_DEF_BEGIN = 34;
+    final static int S_DEF_END = 35;
+    final static int S_BEGIN = 36;
+    final static int S_END = 37;
+    final static int S_AND = 38;
+    final static int S_OR = 39;
+    final static int S_LET = 40;
+    final static int S_DELAY = 41;
+    final static int S_VAR = 42;
 
-    int S_BOOL = 2;
-    int S_NUMBER = 3;
-    int S_CHAR = 4;
-    int S_STRING = 5;
-    int S_ID = 6;
-    int S_LIST_BEGIN = 7;
-    int S_LIST_END = 8;
-    int S_VECTOR_BEGIN = 9;
-    int S_VECTOR_END = 10;
-    int S_LITERAL = 11;
-    int S_QUOT_BEGIN = 12;
-    int S_QUOT_END = 13;
-    int S_CALL = 14;
-    int S_LAMBDA_BEGIN = 15;
-    int S_LAMBDA_END = 16;
-    int S_FORMAL_BEGIN = 17;
-    int S_FORMAL_END = 18;
-    int S_BODY_BEGIN = 19;
-    int S_BODY_END = 20;
-    int S_IF_BEGIN = 21;
-    int S_IF_END = 22;
-    int S_ALTERN = 23;
-    int S_ASSIGN_BEGIN = 24;
-    int S_ASSIGN_END = 25;
-    int S_COND_BEGIN = 26;
-    int S_COND_END = 27;
-    int S_ELSE = 28;
-    int S_CASE_BEGIN = 29;
-    int S_CASE_END = 30;
-    int S_DO_BEGIN = 31;
-    int S_DO_END = 32;
-    int S_COMMAND = 33;
-    int S_DEF_BEGIN = 34;
-    int S_DEF_END = 35;
-    int S_BEGIN = 36;
-    int S_END = 37;
-    int S_AND = 38;
-    int S_OR = 39;
-    int S_LET = 40;
-    int S_DELAY = 41;
-    int S_VAR = 42;
-
-    int NUM_DIFF_TOKENS = 43;
+    final static int NUM_DIFF_TOKENS = 43;
 }

jplag.frontend.text/src/main/antlr/text.g

@@ -58,7 +58,7 @@ options
             InputState state = (InputState) inputState;
             if (text.length() == 0) {
                 // remember token start column
-                state.tokColumn = state.column;
+                state.tokenColumn = state.column;
             }
             state.column++;
         }
@@ -67,7 +67,7 @@ options
 
     protected Token makeToken(int t) {
         ParserToken token = (ParserToken) super.makeToken(t);
-        token.setColumn(((InputState) inputState).tokColumn);
+        token.setColumn(((InputState) inputState).tokenColumn);
         return token;
     }
 }

jplag.frontend.text/src/main/java/de/jplag/text/InputState.java

@@ -12,22 +12,22 @@ import antlr.LexerSharedInputState;
  */
 public class InputState extends LexerSharedInputState {
     public int column = 0;
-    public int tokColumn = 0;
+    public int tokenColumn = 0;
 
-    public InputState(InputBuffer inbuf) {
-        super(inbuf);
+    public InputState(InputBuffer inputBuffer) {
+        super(inputBuffer);
         column = 1;
         line = 1;
     }
 
-    public InputState(InputStream in) {
-        super(in);
+    public InputState(InputStream inputStream) {
+        super(inputStream);
         column = 1;
         line = 1;
     }
 
-    public InputState(Reader in) {
-        super(in);
+    public InputState(Reader inputReader) {
+        super(inputReader);
         column = 1;
         line = 1;
     }

jplag.frontend.text/src/main/java/de/jplag/text/Language.java

@@ -4,7 +4,6 @@ package de.jplag.text;
 import java.io.File;
 
 import de.jplag.ErrorConsumer;
-import de.jplag.Token;
 import de.jplag.TokenList;
 
 public class Language implements de.jplag.Language {
@@ -27,7 +26,7 @@ public class Language implements de.jplag.Language {
 
     @Override
     public String getName() {
-        return "Text AbstractParser";
+        return "Text Parser";
     }
 
     @Override
@@ -69,9 +68,4 @@ public class Language implements de.jplag.Language {
     public int numberOfTokens() {
         return parser.serial;
     }
-
-    @Override
-    public String type2string(int type) {
-        return Token.type2string(type);
-    }
 }

jplag.frontend.text/src/main/java/de/jplag/text/Parser.java

@@ -10,35 +10,51 @@ import de.jplag.AbstractParser;
 import de.jplag.TokenConstants;
 import de.jplag.TokenList;
 
-public class Parser extends AbstractParser implements TokenConstants {
+public class Parser extends AbstractParser {
 
     protected Hashtable<String, Integer> table = new Hashtable<>();
     protected int serial = 1; // 0 is FILE_END token
 
-    private TokenList struct;
+    private TokenList tokens;
 
     private String currentFile;
 
-    public TokenList parse(File dir, String[] files) {
-        struct = new TokenList();
+    private boolean runOut = false;
+
+    public TokenList parse(File directory, String files[]) {
+        tokens = new TokenList();
         errors = 0;
         for (String file : files) {
             getErrorConsumer().print("", "Parsing file " + file);
-            if (!parseFile(dir, file))
+            if (!parseFile(directory, file))
                 errors++;
-            struct.addToken(new TextToken(FILE_END, file, this));
+            tokens.addToken(new TextToken(TokenConstants.FILE_END, file, this));
         }
 
-        TokenList tmp = struct;
-        struct = null;
+        TokenList tmp = tokens;
+        tokens = null;
         this.parseEnd();
         return tmp;
     }
 
-    private boolean parseFile(File dir, String file) {
+    public void add(Token token) {
+        ParserToken parserToken = (ParserToken) token;
+        tokens.addToken(new TextToken(token.getText(), currentFile, parserToken.getLine(), parserToken.getColumn(), parserToken.getLength(), this));
+    }
+
+    public void outOfSerials() {
+        if (runOut)
+            return;
+        runOut = true;
+        errors++;
+        errorConsumer.print("ERROR: Out of serials!", null);
+        System.out.println("de.jplag.text.Parser: ERROR: Out of serials!");
+    }
+
+    private boolean parseFile(File directory, String file) {
         InputState inputState = null;
         try {
-            FileInputStream inputStream = new FileInputStream(new File(dir, file));
+            FileInputStream inputStream = new FileInputStream(new File(directory, file));
             currentFile = file;
             // Create a scanner that reads from the input stream passed to us
             inputState = new InputState(inputStream);
@@ -63,20 +79,4 @@ public class Parser extends AbstractParser implements TokenConstants {
         }
         return true;
     }
-
-    public void add(Token tok) {
-        ParserToken ptok = (ParserToken) tok;
-        struct.addToken(new TextToken(tok.getText(), currentFile, ptok.getLine(), ptok.getColumn(), ptok.getLength(), this));
-    }
-
-    private boolean runOut = false;
-
-    public void outOfSerials() {
-        if (runOut)
-            return;
-        runOut = true;
-        errors++;
-        errorConsumer.print("ERROR: Out of serials!", null);
-        System.out.println("de.jplag.text.Parser: ERROR: Out of serials!");
-    }
 }

jplag.frontend.text/src/main/java/de/jplag/text/ParserToken.java

@@ -6,23 +6,22 @@ public class ParserToken extends Token {
     /**
      * This variable holds the line number of the current token.
      */
-    private int _line = -1;
+    private int line = -1;
 
     /**
      * This variable holds the column of the current token in its line.
      */
-    private int _column = -1;
+    private int column = -1;
 
     /**
      * This variable holds the label of the current token.
      */
-    private String _text = null;
+    private String text = null;
 
     /**
      * This variable holds the identifier of the current token.
      */
-
-    private int _id = -1;
+    private int id = -1;
 
     public ParserToken() {
         super();
@@ -30,44 +29,44 @@ public class ParserToken extends Token {
 
     @Override
     public void setLine(int line) {
-        _line = line;
+        this.line = line;
     }
 
     @Override
     public void setColumn(int column) {
-        _column = column;
+        this.column = column;
     }
 
     public void setID(int id) {
-        _id = id;
+        this.id = id;
     }
 
     @Override
     public void setText(String text) {
-        _text = (text != null ? text.intern() : null);
+        this.text = (text != null ? text.intern() : null);
     }
 
     @Override
     public int getColumn() {
-        return _column;
+        return column;
     }
 
     @Override
     public int getLine() {
-        return _line;
+        return line;
     }
 
     @Override
     public String getText() {
-        return _text;
+        return text;
     }
 
     public int getID() {
-        return _id;
+        return id;
     }
 
     public int getLength() {
-        return _text.length();
+        return text.length();
     }
 
     @Override

jplag.frontend.text/src/main/java/de/jplag/text/TextToken.java

@@ -22,9 +22,6 @@ public class TextToken extends Token {
         return serial;
     }
 
-    // ///////////////////// END OF STATIC MEMBERS
-
-    private int line, column, length;
     private String text;
 
     public TextToken(int type, String file, Parser parser) {
@@ -37,37 +34,12 @@ public class TextToken extends Token {
         this.text = text.toLowerCase();
     }
 
-    @Override
-    public int getLine() {
-        return line;
-    }
-
-    @Override
-    public int getColumn() {
-        return column;
-    }
-
-    @Override
-    public int getLength() {
-        return length;
-    }
-
-    @Override
-    public void setLine(int line) {
-        this.line = line;
-    }
-
-    @Override
-    public void setColumn(int column) {
-        this.column = column;
+    public String getText() {
+        return this.text;
     }
 
     @Override
-    public void setLength(int length) {
-        this.length = length;
-    }
-
-    public String getText() {
-        return this.text;
+    protected String type2string() {
+        return getText();
     }
 }

jplag/pom.xml

@@ -20,6 +20,22 @@
             <artifactId>system-rules</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.13.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-math3</artifactId>
+            <version>3.6.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>4.0.0</version>
+            <scope>test</scope>
+        </dependency>
 
         <dependency>
             <groupId>de.jplag</groupId>

jplag/src/main/java/de/jplag/CLI.java

@@ -1,20 +1,8 @@
 package de.jplag;
 
-import static de.jplag.CommandLineArgument.BASE_CODE;
-import static de.jplag.CommandLineArgument.COMPARISON_MODE;
-import static de.jplag.CommandLineArgument.DEBUG;
-import static de.jplag.CommandLineArgument.EXCLUDE_FILE;
-import static de.jplag.CommandLineArgument.LANGUAGE;
-import static de.jplag.CommandLineArgument.MIN_TOKEN_MATCH;
-import static de.jplag.CommandLineArgument.RESULT_FOLDER;
-import static de.jplag.CommandLineArgument.ROOT_DIRECTORY;
-import static de.jplag.CommandLineArgument.SHOWN_COMPARISONS;
-import static de.jplag.CommandLineArgument.SIMILARITY_THRESHOLD;
-import static de.jplag.CommandLineArgument.SUBDIRECTORY;
-import static de.jplag.CommandLineArgument.SUFFIXES;
-import static de.jplag.CommandLineArgument.VERBOSITY;
-
-import java.io.File;
+import static de.jplag.CommandLineArgument.*;
+
+import java.util.Optional;
 import java.util.Random;
 
 import net.sourceforge.argparse4j.ArgumentParsers;
@@ -22,11 +10,17 @@ import net.sourceforge.argparse4j.inf.ArgumentParser;
 import net.sourceforge.argparse4j.inf.ArgumentParserException;
 import net.sourceforge.argparse4j.inf.Namespace;
 
+import de.jplag.clustering.ClusteringAlgorithm;
+import de.jplag.clustering.ClusteringOptions;
+import de.jplag.clustering.Preprocessing;
+import de.jplag.clustering.algorithm.InterClusterSimilarity;
 import de.jplag.exceptions.ExitException;
 import de.jplag.options.JPlagOptions;
 import de.jplag.options.LanguageOption;
+import de.jplag.options.SimilarityMetric;
 import de.jplag.options.Verbosity;
-import de.jplag.reporting.Report;
+import de.jplag.reporting2.JsonReport;
+import de.jplag.reporting2.Report;
 import de.jplag.strategy.ComparisonMode;
 
 /**
@@ -42,6 +36,8 @@ public class CLI {
             "More Abstract than Tree", "Students Nightmare", "No, changing variable names does not work", "The tech is out there!"};
 
     private static final String PROGRAM_NAME = "jplag";
+    static final String CLUSTERING_GROUP_NAME = "Clustering";
+    static final String CLUSTERING_PREPROCESSING_GROUP_NAME = "Clustering - Preprocessing";
 
     private final ArgumentParser parser;
 
@@ -57,9 +53,8 @@ public class CLI {
             JPlag program = new JPlag(options);
             System.out.println("JPlag initialized");
             JPlagResult result = program.run();
-            File reportDir = new File(arguments.getString(RESULT_FOLDER.flagWithoutDash()));
-            Report report = new Report(reportDir, options);
-            report.writeResult(result);
+            Report report = new JsonReport();
+            report.saveReport(result, arguments.getString(RESULT_FOLDER.flagWithoutDash()));
         } catch (ExitException exception) {
             System.out.println("Error: " + exception.getMessage());
             System.exit(1);
@@ -71,8 +66,9 @@ public class CLI {
      */
     public CLI() {
         parser = ArgumentParsers.newFor(PROGRAM_NAME).build().defaultHelp(true).description(generateDescription());
+        CliGroupHelper groupHelper = new CliGroupHelper(parser);
         for (CommandLineArgument argument : CommandLineArgument.values()) {
-            argument.parseWith(parser);
+            argument.parseWith(parser, groupHelper);
         }
     }
 
@@ -115,6 +111,40 @@ public class CLI {
         options.setMaximumNumberOfComparisons(SHOWN_COMPARISONS.getFrom(namespace));
         ComparisonMode.fromName(COMPARISON_MODE.getFrom(namespace)).ifPresentOrElse(options::setComparisonMode,
                 () -> System.out.println("Unknown comparison mode, using default mode!"));
+
+        ClusteringOptions.Builder clusteringBuilder = new ClusteringOptions.Builder();
+        Optional.ofNullable((Boolean) CLUSTER_ENABLE.getFrom(namespace)).ifPresent(clusteringBuilder::enabled);
+        Optional.ofNullable((ClusteringAlgorithm) CLUSTER_ALGORITHM.getFrom(namespace)).ifPresent(clusteringBuilder::algorithm);
+        Optional.ofNullable((SimilarityMetric) CLUSTER_METRIC.getFrom(namespace)).ifPresent(clusteringBuilder::similarityMetric);
+        Optional.ofNullable((Float) CLUSTER_SPECTRAL_BANDWIDTH.getFrom(namespace)).ifPresent(clusteringBuilder::spectralKernelBandwidth);
+        Optional.ofNullable((Float) CLUSTER_SPECTRAL_NOISE.getFrom(namespace)).ifPresent(clusteringBuilder::spectralGaussianProcessVariance);
+        Optional.ofNullable((Integer) CLUSTER_SPECTRAL_MIN_RUNS.getFrom(namespace)).ifPresent(clusteringBuilder::spectralMinRuns);
+        Optional.ofNullable((Integer) CLUSTER_SPECTRAL_MAX_RUNS.getFrom(namespace)).ifPresent(clusteringBuilder::spectralMaxRuns);
+        Optional.ofNullable((Integer) CLUSTER_SPECTRAL_KMEANS_ITERATIONS.getFrom(namespace))
+                .ifPresent(clusteringBuilder::spectralMaxKMeansIterationPerRun);
+        Optional.ofNullable((Float) CLUSTER_AGGLOMERATIVE_THRESHOLD.getFrom(namespace)).ifPresent(clusteringBuilder::agglomerativeThreshold);
+        Optional.ofNullable((InterClusterSimilarity) CLUSTER_AGGLOMERATIVE_INTER_CLUSTER_SIMILARITY.getFrom(namespace))
+                .ifPresent(clusteringBuilder::agglomerativeInterClusterSimilarity);
+        Optional.ofNullable((Boolean) CLUSTER_PREPROCESSING_NONE.getFrom(namespace)).ifPresent(none -> {
+            if (none) {
+                clusteringBuilder.preprocessor(Preprocessing.NONE);
+            }
+        });
+        Optional.ofNullable((Boolean) CLUSTER_PREPROCESSING_CDF.getFrom(namespace)).ifPresent(cdf -> {
+            if (cdf) {
+                clusteringBuilder.preprocessor(Preprocessing.CUMULATIVE_DISTRIBUTION_FUNCTION);
+            }
+        });
+        Optional.ofNullable((Float) CLUSTER_PREPROCESSING_PERCENTILE.getFrom(namespace)).ifPresent(percentile -> {
+            clusteringBuilder.preprocessor(Preprocessing.PERCENTILE);
+            clusteringBuilder.preprocessorPercentile(percentile);
+        });
+        Optional.ofNullable((Float) CLUSTER_PREPROCESSING_THRESHOLD.getFrom(namespace)).ifPresent(threshold -> {
+            clusteringBuilder.preprocessor(Preprocessing.THRESHOLD);
+            clusteringBuilder.preprocessorPercentile(threshold);
+        });
+        options.setClusteringOptions(clusteringBuilder.build());
+
         return options;
     }
 

jplag/src/main/java/de/jplag/CliGroupHelper.java

@@ -0,0 +1,33 @@
+package de.jplag;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import net.sourceforge.argparse4j.inf.ArgumentContainer;
+import net.sourceforge.argparse4j.inf.ArgumentGroup;
+import net.sourceforge.argparse4j.inf.ArgumentParser;
+import net.sourceforge.argparse4j.inf.MutuallyExclusiveGroup;
+
+/**
+ * Can be used to automatically create and reuse {@link ArgumentGroup}s and {@link MutuallyExclusiveGroup}s through
+ * their names only. This is useful when an {@link ArgumentParser} is not configured in an imperative fashion.
+ */
+public class CliGroupHelper {
+
+    private final ArgumentParser parser;
+    private Map<String, MutuallyExclusiveGroup> mutuallyExclusiveGroups = new HashMap<>();
+    private Map<String, ArgumentGroup> argumentGroups = new HashMap<>();
+
+    public CliGroupHelper(ArgumentParser parser) {
+        this.parser = parser;
+    }
+
+    public ArgumentContainer getMutuallyExclusiveGroup(String name) {
+        return mutuallyExclusiveGroups.computeIfAbsent(name, parser::addMutuallyExclusiveGroup);
+    }
+
+    public ArgumentContainer getArgumentGroup(String name) {
+        return argumentGroups.computeIfAbsent(name, parser::addArgumentGroup);
+    }
+
+}

jplag/src/main/java/de/jplag/CommandLineArgument.java

@@ -1,5 +1,7 @@
 package de.jplag;
 
+import static de.jplag.CLI.CLUSTERING_GROUP_NAME;
+import static de.jplag.CLI.CLUSTERING_PREPROCESSING_GROUP_NAME;
 import static de.jplag.options.JPlagOptions.DEFAULT_COMPARISON_MODE;
 import static de.jplag.options.JPlagOptions.DEFAULT_SHOWN_COMPARISONS;
 import static de.jplag.options.JPlagOptions.DEFAULT_SIMILARITY_THRESHOLD;
@@ -9,11 +11,18 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
 
+import net.sourceforge.argparse4j.impl.Arguments;
 import net.sourceforge.argparse4j.inf.Argument;
+import net.sourceforge.argparse4j.inf.ArgumentAction;
+import net.sourceforge.argparse4j.inf.ArgumentContainer;
 import net.sourceforge.argparse4j.inf.ArgumentParser;
 import net.sourceforge.argparse4j.inf.Namespace;
 
+import de.jplag.clustering.ClusteringAlgorithm;
+import de.jplag.clustering.ClusteringOptions;
+import de.jplag.clustering.algorithm.InterClusterSimilarity;
 import de.jplag.options.LanguageOption;
+import de.jplag.options.SimilarityMetric;
 import de.jplag.strategy.ComparisonMode;
 
 /**
@@ -21,50 +30,84 @@ import de.jplag.strategy.ComparisonMode;
  * @author Timur Saglam
  */
 public enum CommandLineArgument {
-    ROOT_DIRECTORY("rootDir", NumberOfArgumentValues.ONE_OR_MORE_VALUES, String.class),
-    LANGUAGE("-l", String.class, LanguageOption.getDefault().getDisplayName(), LanguageOption.getAllDisplayNames()),
+
+    ROOT_DIRECTORY(new Builder("rootDir", String.class).nargs(NumberOfArgumentValues.ONE_OR_MORE_VALUES)),
+    LANGUAGE(new Builder("-l", String.class).defaultsTo(LanguageOption.getDefault().getDisplayName()).choices(LanguageOption.getAllDisplayNames())),
     BASE_CODE("-bc", String.class),
-    VERBOSITY("-v", String.class, "quiet", List.of("quiet", "long")), // TODO SH: Replace verbosity when integrating a real logging library
+    VERBOSITY(new Builder("-v", String.class).defaultsTo("quiet").choices(List.of("quiet", "long"))), // TODO SH: Replace verbosity when integrating a
+                                                                                                      // real logging library
     DEBUG("-d", Boolean.class),
     SUBDIRECTORY("-S", String.class),
     SUFFIXES("-p", String.class),
     EXCLUDE_FILE("-x", String.class),
     MIN_TOKEN_MATCH("-t", Integer.class),
-    SIMILARITY_THRESHOLD("-m", Float.class, DEFAULT_SIMILARITY_THRESHOLD),
-    SHOWN_COMPARISONS("-n", Integer.class, DEFAULT_SHOWN_COMPARISONS),
-    RESULT_FOLDER("-r", String.class, "result"),
-    COMPARISON_MODE("-c", String.class, DEFAULT_COMPARISON_MODE.getName(), ComparisonMode.allNames());
+    SIMILARITY_THRESHOLD(new Builder("-m", Float.class).defaultsTo(DEFAULT_SIMILARITY_THRESHOLD)),
+    SHOWN_COMPARISONS(new Builder("-n", Integer.class).defaultsTo(DEFAULT_SHOWN_COMPARISONS)),
+    RESULT_FOLDER(new Builder("-r", String.class).defaultsTo("result")),
+    COMPARISON_MODE(new Builder("-c", String.class).defaultsTo(DEFAULT_COMPARISON_MODE.getName()).choices(ComparisonMode.allNames())),
+    CLUSTER_ENABLE(new Builder("--cluster-skip", Boolean.class).argumentGroup(CLUSTERING_GROUP_NAME).action(Arguments.storeTrue())),
+    CLUSTER_ALGORITHM(
+            new Builder("--cluster-alg", ClusteringAlgorithm.class).argumentGroup(CLUSTERING_GROUP_NAME)
+                    .defaultsTo(ClusteringOptions.DEFAULTS.getAlgorithm())),
+    CLUSTER_METRIC(
+            new Builder("--cluster-metric", SimilarityMetric.class).argumentGroup(CLUSTERING_GROUP_NAME)
+                    .defaultsTo(ClusteringOptions.DEFAULTS.getSimilarityMetric())),
+    CLUSTER_SPECTRAL_BANDWIDTH(
+            new Builder("--cluster-spectral-bandwidth", Float.class).argumentGroup(CLUSTERING_GROUP_NAME).metaVar("bandwidth")
+                    .defaultsTo(ClusteringOptions.DEFAULTS.getSpectralKernelBandwidth())),
+    CLUSTER_SPECTRAL_NOISE(
+            new Builder("--cluster-spectral-noise", Float.class).argumentGroup(CLUSTERING_GROUP_NAME).metaVar("noise")
+                    .defaultsTo(ClusteringOptions.DEFAULTS.getSpectralGaussianProcessVariance())),
+    CLUSTER_SPECTRAL_MIN_RUNS(
+            new Builder("--cluster-spectral-min-runs", Integer.class).argumentGroup(CLUSTERING_GROUP_NAME).metaVar("min")
+                    .defaultsTo(ClusteringOptions.DEFAULTS.getSpectralMinRuns())),
+    CLUSTER_SPECTRAL_MAX_RUNS(
+            new Builder("--cluster-spectral-max-runs", Integer.class).argumentGroup(CLUSTERING_GROUP_NAME).metaVar("max")
+                    .defaultsTo(ClusteringOptions.DEFAULTS.getSpectralMaxRuns())),
+    CLUSTER_SPECTRAL_KMEANS_ITERATIONS(
+            new Builder("--cluster-spectral-kmeans-interations", Integer.class).argumentGroup(CLUSTERING_GROUP_NAME).metaVar("iterations")
+                    .defaultsTo(ClusteringOptions.DEFAULTS.getSpectralMaxKMeansIterationPerRun())),
+    CLUSTER_AGGLOMERATIVE_THRESHOLD(
+            new Builder("--cluster-agglomerative-threshold", Float.class).argumentGroup(CLUSTERING_GROUP_NAME).metaVar("threshold")
+                    .defaultsTo(ClusteringOptions.DEFAULTS.getAgglomerativeThreshold())),
+    CLUSTER_AGGLOMERATIVE_INTER_CLUSTER_SIMILARITY(
+            new Builder("--cluster-agglomerative-inter-cluster-similarity", InterClusterSimilarity.class).argumentGroup(CLUSTERING_GROUP_NAME)
+                    .defaultsTo(ClusteringOptions.DEFAULTS.getAgglomerativeInterClusterSimilarity())),
+    CLUSTER_PREPROCESSING_NONE(
+            new Builder("--cluster-pp-none", Boolean.class).mutuallyExclusiveGroup(CLUSTERING_PREPROCESSING_GROUP_NAME)
+                    .action(Arguments.storeTrue())),
+    CLUSTER_PREPROCESSING_CDF(
+            new Builder("--cluster-pp-cdf", Boolean.class).mutuallyExclusiveGroup(CLUSTERING_PREPROCESSING_GROUP_NAME).action(Arguments.storeTrue())),
+    CLUSTER_PREPROCESSING_PERCENTILE(
+            new Builder("--cluster-pp-percentile", Float.class).mutuallyExclusiveGroup(CLUSTERING_PREPROCESSING_GROUP_NAME).metaVar("percentile")),
+    CLUSTER_PREPROCESSING_THRESHOLD(
+            new Builder("--cluster-pp-threshold", Float.class).mutuallyExclusiveGroup(CLUSTERING_PREPROCESSING_GROUP_NAME).metaVar("threshold"));
 
     private final String flag;
     private final NumberOfArgumentValues numberOfValues;
     private final String description;
     private final Optional<Object> defaultValue;
     private final Optional<Collection<String>> choices;
+    private final Optional<String> argumentGroup;
+    private final Optional<String> mutuallyExclusiveGroup;
+    private final Optional<ArgumentAction> action;
+    private final Optional<String> metaVar;
     private final Class<?> type;
 
-    CommandLineArgument(String flag, Class<?> type) {
-        this(flag, NumberOfArgumentValues.SINGLE_VALUE, type, Optional.empty(), Optional.empty());
-    }
-
-    CommandLineArgument(String flag, NumberOfArgumentValues numberOfValues, Class<?> type) {
-        this(flag, numberOfValues, type, Optional.empty(), Optional.empty());
-    }
-
-    CommandLineArgument(String flag, Class<?> type, Object defaultValue) {
-        this(flag, NumberOfArgumentValues.SINGLE_VALUE, type, Optional.of(defaultValue), Optional.empty());
-    }
-
-    CommandLineArgument(String flag, Class<?> type, Object defaultValue, Collection<String> choices) {
-        this(flag, NumberOfArgumentValues.SINGLE_VALUE, type, Optional.of(defaultValue), Optional.of(choices));
+    private CommandLineArgument(String flag, Class<?> type) {
+        this(new Builder(flag, type));
     }
 
-    CommandLineArgument(String flag, NumberOfArgumentValues numberOfValues, Class<?> type, Optional<Object> defaultValue,
-            Optional<Collection<String>> choices) {
-        this.flag = flag;
-        this.numberOfValues = numberOfValues;
-        this.type = type;
-        this.defaultValue = defaultValue;
-        this.choices = choices;
+    private CommandLineArgument(Builder builder) {
+        this.flag = builder.flag;
+        this.type = builder.type;
+        this.defaultValue = builder.defaultValue;
+        this.choices = builder.choices;
+        this.argumentGroup = builder.argumentGroup;
+        this.mutuallyExclusiveGroup = builder.mutuallyExclusiveGroup;
+        this.action = builder.action;
+        this.metaVar = builder.metaVar;
+        this.numberOfValues = builder.nargs.orElse(NumberOfArgumentValues.SINGLE_VALUE);
         this.description = retrieveDescriptionFromMessages();
     }
 
@@ -76,10 +119,10 @@ public enum CommandLineArgument {
     }
 
     /**
-     * @return the flag name of the command line argument without leading dashes.
+     * @return the flag name of the command line argument without leading dashes and inner dashes replaced with underscores.
      */
     public String flagWithoutDash() {
-        return flag.replace("-", "");
+        return flag.replaceAll("^-+", "").replaceAll("-", "_");
     }
 
     /**
@@ -111,10 +154,15 @@ public enum CommandLineArgument {
      * Parses the command line argument with a specific parser.
      * @param parser is that parser.
      */
-    public void parseWith(ArgumentParser parser) {
-        Argument argument = parser.addArgument(flag).help(description);
-        choices.ifPresent(argument::choices);
-        defaultValue.ifPresent(argument::setDefault);
+    public void parseWith(ArgumentParser parser, CliGroupHelper groupHelper) {
+        ArgumentContainer argContainer = mutuallyExclusiveGroup.map(groupHelper::getMutuallyExclusiveGroup)
+                .or(() -> argumentGroup.map(groupHelper::getArgumentGroup)).orElse(parser);
+
+        Argument argument = argContainer.addArgument(flag).help(description);
+        choices.ifPresent(it -> argument.choices(it));
+        defaultValue.ifPresent(it -> argument.setDefault(it));
+        action.ifPresent(argument::action);
+        metaVar.ifPresent(argument::metavar);
         argument.type(type);
         if (type == Boolean.class) {
             argument.action(storeTrue());
@@ -136,4 +184,56 @@ public enum CommandLineArgument {
         }
         return Messages.getString(getClass().getSimpleName() + "." + builder.toString());
     }
+
+    private static class Builder {
+        private final String flag;
+        private final Class<?> type;
+        private Optional<Object> defaultValue = Optional.empty();
+        private Optional<Collection<String>> choices = Optional.empty();
+        private Optional<String> argumentGroup = Optional.empty();
+        private Optional<String> mutuallyExclusiveGroup = Optional.empty();
+        private Optional<ArgumentAction> action = Optional.empty();
+        private Optional<String> metaVar = Optional.empty();
+        private Optional<NumberOfArgumentValues> nargs = Optional.empty();
+
+        public Builder(String flag, Class<?> type) {
+            this.flag = flag;
+            this.type = type;
+        }
+
+        public Builder defaultsTo(Object defaultValue) {
+            this.defaultValue = Optional.of(defaultValue);
+            return this;
+        }
+
+        public Builder choices(Collection<String> choices) {
+            this.choices = Optional.of(choices);
+            return this;
+        }
+
+        public Builder argumentGroup(String argumentGroup) {
+            this.argumentGroup = Optional.of(argumentGroup);
+            return this;
+        }
+
+        public Builder mutuallyExclusiveGroup(String mutuallyExclusiveGroup) {
+            this.mutuallyExclusiveGroup = Optional.of(mutuallyExclusiveGroup);
+            return this;
+        }
+
+        public Builder action(ArgumentAction action) {
+            this.action = Optional.of(action);
+            return this;
+        }
+
+        public Builder metaVar(String metaVar) {
+            this.metaVar = Optional.of(metaVar);
+            return this;
+        }
+
+        public Builder nargs(NumberOfArgumentValues nargs) {
+            this.nargs = Optional.of(nargs);
+            return this;
+        }
+    }
 }

jplag/src/main/java/de/jplag/GreedyStringTiling.java

@@ -1,5 +1,8 @@
 package de.jplag;
 
+import static de.jplag.TokenConstants.FILE_END;
+import static de.jplag.TokenConstants.SEPARATOR_TOKEN;
+
 import java.util.ArrayList;
 import java.util.List;
 
@@ -13,7 +16,7 @@ import de.jplag.options.JPlagOptions;
  * "https://www.researchgate.net/publication/262763983_String_Similarity_via_Greedy_String_Tiling_and_Running_Karp-Rabin_Matching">
  * String Similarity via Greedy String Tiling and Running Karp−Rabin Matching </a>
  */
-public class GreedyStringTiling implements TokenConstants {
+public class GreedyStringTiling {
 
     private final JPlagOptions options;
 
@@ -49,7 +52,7 @@ public class GreedyStringTiling implements TokenConstants {
         for (int i = 0; i < hashLength; i++) {
             hash = (2 * hash) + (tokenList.getToken(i).type & modulo);
             hashedLength++;
-            if (tokenList.getToken(i).marked) {
+            if (tokenList.getToken(i).isMarked()) {
                 hashedLength = 0;
             }
         }
@@ -58,14 +61,14 @@ public class GreedyStringTiling implements TokenConstants {
         if (makeTable) {
             for (int i = 0; i < loops; i++) {
                 if (hashedLength >= hashLength) {
-                    tokenList.getToken(i).hash = hash;
+                    tokenList.getToken(i).setHash(hash);
                     tokenList.tokenHashes.put(hash, i);   // add into hashtable
                 } else {
-                    tokenList.getToken(i).hash = -1;
+                    tokenList.getToken(i).setHash(-1);
                 }
                 hash -= factor * (tokenList.getToken(i).type & modulo);
                 hash = (2 * hash) + (tokenList.getToken(i + hashLength).type & modulo);
-                if (tokenList.getToken(i + hashLength).marked) {
+                if (tokenList.getToken(i + hashLength).isMarked()) {
                     hashedLength = 0;
                 } else {
                     hashedLength++;
@@ -73,10 +76,10 @@ public class GreedyStringTiling implements TokenConstants {
             }
         } else {
             for (int i = 0; i < loops; i++) {
-                tokenList.getToken(i).hash = (hashedLength >= hashLength) ? hash : -1;
+                tokenList.getToken(i).setHash((hashedLength >= hashLength) ? hash : -1);
                 hash -= factor * (tokenList.getToken(i).type & modulo);
                 hash = (2 * hash) + (tokenList.getToken(i + hashLength).type & modulo);
-                if (tokenList.getToken(i + hashLength).marked) {
+                if (tokenList.getToken(i + hashLength).isMarked()) {
                     hashedLength = 0;
                 } else {
                     hashedLength++;
@@ -151,27 +154,27 @@ public class GreedyStringTiling implements TokenConstants {
             maxMatch = minimumTokenMatch;
             matches.clear();
             for (int x = 0; x < first.size() - maxMatch; x++) {
-                List<Integer> hashedTokens = second.tokenHashes.get(first.getToken(x).hash);
-                if (first.getToken(x).marked || first.getToken(x).hash == -1) {
+                List<Integer> hashedTokens = second.tokenHashes.get(first.getToken(x).getHash());
+                if (first.getToken(x).isMarked() || first.getToken(x).getHash() == -1) {
                     continue;
                 }
                 inner: for (Integer y : hashedTokens) {
-                    if (second.getToken(y).marked || maxMatch >= second.size() - y) { // >= because of pivots!
+                    if (second.getToken(y).isMarked() || maxMatch >= second.size() - y) { // >= because of pivots!
                         continue;
                     }
 
                     int j, hx, hy;
                     for (j = maxMatch - 1; j >= 0; j--) { // begins comparison from behind
-                        if (first.getToken(hx = x + j).type != second.getToken(hy = y + j).type || first.getToken(hx).marked
-                                || second.getToken(hy).marked) {
+                        if (first.getToken(hx = x + j).type != second.getToken(hy = y + j).type || first.getToken(hx).isMarked()
+                                || second.getToken(hy).isMarked()) {
                             continue inner;
                         }
                     }
 
                     // expand match
                     j = maxMatch;
-                    while (first.getToken(hx = x + j).type == second.getToken(hy = y + j).type && !first.getToken(hx).marked
-                            && !second.getToken(hy).marked) {
+                    while (first.getToken(hx = x + j).type == second.getToken(hy = y + j).type && !first.getToken(hx).isMarked()
+                            && !second.getToken(hy).isMarked()) {
                         j++;
                     }
 
@@ -189,9 +192,11 @@ public class GreedyStringTiling implements TokenConstants {
                 comparison.addMatch(x, y, matches.get(i).getLength());
                 // in order that "Match" will be newly build (because reusing)
                 for (int j = matches.get(i).getLength(); j > 0; j--) {
-                    first.getToken(x).marked = second.getToken(y).marked = true; // mark all Tokens!
+                    first.getToken(x).setMarked(true); // mark all Tokens!
+                    second.getToken(y).setMarked(true);
                     if (isBaseCodeComparison) {
-                        first.getToken(x).basecode = second.getToken(y).basecode = true;
+                        first.getToken(x).setBasecode(true);
+                        second.getToken(y).setBasecode(true);
                     }
                     x++;
                     y++;
@@ -220,9 +225,9 @@ public class GreedyStringTiling implements TokenConstants {
     private void markTokens(TokenList tokenList, boolean isBaseCodeComparison) {
         for (Token token : tokenList.allTokens()) {
             if (isBaseCodeComparison) {
-                token.marked = token.type == FILE_END || token.type == SEPARATOR_TOKEN;
+                token.setMarked(token.type == FILE_END || token.type == SEPARATOR_TOKEN);
             } else {
-                token.marked = token.type == FILE_END || token.type == SEPARATOR_TOKEN || (token.basecode && options.hasBaseCode());
+                token.setMarked(token.type == FILE_END || token.type == SEPARATOR_TOKEN || (token.isBasecode() && options.hasBaseCode()));
             }
         }
     }

jplag/src/main/java/de/jplag/JPlag.java

@@ -12,6 +12,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import de.jplag.clustering.ClusteringFactory;
 import de.jplag.exceptions.ExitException;
 import de.jplag.exceptions.SubmissionException;
 import de.jplag.options.JPlagOptions;
@@ -89,6 +90,9 @@ public class JPlag {
         // Compare valid submissions.
         JPlagResult result = comparisonStrategy.compareSubmissions(submissionSet);
         errorCollector.print("\nTotal time for comparing submissions: " + TimeUtil.formatDuration(result.getDuration()), null);
+
+        result.setClusteringResult(ClusteringFactory.getClusterings(result.getComparisons(), options.getClusteringOptions()));
+
         return result;
     }
 

jplag/src/main/java/de/jplag/JPlagComparison.java

@@ -78,7 +78,7 @@ public class JPlagComparison implements Comparator<JPlagComparison> { // FIXME T
          */
         Set<String> collectedFiles = new LinkedHashSet<>();
         for (Match match : matches) {
-            collectedFiles.add(tokenList.getToken(match.getStart(j == 0)).file);
+            collectedFiles.add(tokenList.getToken(match.getStart(j == 0)).getFile());
         }
 
         /*

jplag/src/main/java/de/jplag/JPlagResult.java

@@ -2,6 +2,7 @@ package de.jplag;
 
 import java.util.List;
 
+import de.jplag.clustering.ClusteringResult;
 import de.jplag.options.JPlagOptions;
 
 /**
@@ -19,6 +20,8 @@ public class JPlagResult {
 
     private final int[] similarityDistribution; // 10-element array representing the similarity distribution of the detected matches.
 
+    private List<ClusteringResult<Submission>> clusteringResult;
+
     public JPlagResult(List<JPlagComparison> comparisons, SubmissionSet submissions, long durationInMillis, JPlagOptions options) {
         this.comparisons = comparisons;
         this.submissions = submissions;
@@ -37,6 +40,10 @@ public class JPlagResult {
         this.comparisons = this.getComparisons(limit);
     }
 
+    public void setClusteringResult(List<ClusteringResult<Submission>> clustering) {
+        this.clusteringResult = clustering;
+    }
+
     /**
      * @return a list of all comparisons sorted by percentage (descending)
      */
@@ -95,6 +102,10 @@ public class JPlagResult {
         return similarityDistribution;
     }
 
+    public List<ClusteringResult<Submission>> getClusteringResult() {
+        return this.clusteringResult;
+    }
+
     @Override
     public String toString() {
         return String.format("JPlagResult { comparisons: %d, duration: %d ms, language: %s, submissions: %d }", getComparisons().size(),

jplag/src/main/java/de/jplag/Submission.java

@@ -267,7 +267,7 @@ public class Submission implements Comparable<Submission> {
      */
     public void resetBaseCode() {
         for (Token token : tokenList.allTokens()) {
-            token.basecode = false;
+            token.setBasecode(false);
         }
     }
 

jplag/src/main/java/de/jplag/clustering/Cluster.java

@@ -0,0 +1,115 @@
+package de.jplag.clustering;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.BiFunction;
+import java.util.stream.Collectors;
+
+/**
+ * Cluster part of a {@link ClusteringResult}.
+ * @param <T> type of the clusters members
+ */
+public class Cluster<T> {
+
+    private float communityStrength;
+    private Collection<T> members;
+    private ClusteringResult<T> clusteringResult = null;
+
+    public Cluster(Collection<T> members, float communityStrength) {
+        this.members = new ArrayList<>(members);
+        this.communityStrength = communityStrength;
+    }
+
+    public Collection<T> getMembers() {
+        return members;
+    }
+
+    /**
+     * See {@link ClusteringResult#getCommunityStrength}
+     * @return community strength of the cluster
+     */
+    public float getCommunityStrength() {
+        return communityStrength;
+    }
+
+    /**
+     * Sets this clusters clustering result. Should only be called by classes extending {@link ClusteringResult} on their
+     * own clusters.
+     * @param clusteringResult the clustering result
+     */
+    public void setClusteringResult(ClusteringResult<T> clusteringResult) {
+        this.clusteringResult = clusteringResult;
+    }
+
+    /**
+     * @return How much each member of this cluster contributes to the {@link ClusteringResult#getCommunityStrength}.
+     */
+    public float getCommunityStrengthPerConnection() {
+        int size = getMembers().size();
+        if (size < 2)
+            return 0;
+        return getCommunityStrength() / connections();
+    }
+
+    /**
+     * Computes a normalized community strength per connection. Can be used as measure for strength of evidence in
+     * comparison to other clusters in the same clustering. Guaranteed to be smaller than 1. Negative values indicate
+     * non-clusters. This method may only be called on clusters that are part of a ClusteringResult.
+     * @return normalized community strength per connection
+     */
+    public float getNormalizedCommunityStrengthPerConnection() {
+        List<Cluster<T>> goodClusters = clusteringResult.getClusters().stream().filter(cluster -> cluster.getCommunityStrength() > 0)
+                .collect(Collectors.toList());
+        float posCommunityStrengthSum = (float) goodClusters.stream().mapToDouble(Cluster::getCommunityStrengthPerConnection).sum();
+
+        int size = clusteringResult.getClusters().size();
+        if (size < 2)
+            return getCommunityStrengthPerConnection();
+        return getCommunityStrengthPerConnection() / posCommunityStrengthSum;
+    }
+
+    /**
+     * How much this cluster is worth during optimization.
+     */
+    public double getWorth(BiFunction<T, T, Float> similarity) {
+        double communityStrength = getCommunityStrength();
+        if (getMembers().size() > 1) {
+            communityStrength /= connections();
+        }
+        double averageSimilarity = averageSimilarity(similarity);
+        return communityStrength * averageSimilarity;
+    }
+
+    /**
+     * Computes the average similarity inside the cluster.
+     * @param similarity function that supplies the similarity of two cluster members.
+     * @return average similarity
+     */
+    public float averageSimilarity(BiFunction<T, T, Float> similarity) {
+        List<T> members = new ArrayList<>(getMembers());
+        if (members.size() < 2) {
+            return 1;
+        }
+        float similaritySum = 0;
+        for (int i = 0; i < members.size(); i++) {
+            for (int j = i + 1; j < members.size(); j++) {
+                similaritySum += similarity.apply(members.get(i), members.get(j));
+            }
+        }
+        return similaritySum / connections();
+    }
+
+    private int connections() {
+        int size = getMembers().size();
+        return ((size - 1) * size) / 2;
+    }
+
+    /**
+     * Whether this cluster is very uninformative or wrong and should be pruned as last step of the clustering process.
+     * @return is bad
+     */
+    public boolean isBadCluster() {
+        return getMembers().size() < 2 || getCommunityStrength() < 0;
+    }
+}

jplag/src/main/java/de/jplag/clustering/ClusteringAdapter.java

@@ -0,0 +1,68 @@
+package de.jplag.clustering;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.linear.RealMatrix;
+
+import de.jplag.JPlagComparison;
+import de.jplag.Submission;
+import de.jplag.clustering.algorithm.GenericClusteringAlgorithm;
+
+/**
+ * This class acts as an adapter between
+ * <ul>
+ * <li>the clustering algorithms (that operate on collections of integers)</li>
+ * <li>and the rest of the code base (that operates on {@link ClusteringResult}s of {@link Submission}s)</li>
+ * </ul>
+ */
+public class ClusteringAdapter {
+
+    private RealMatrix similarityMatrix;
+    private IntegerMapping<Submission> mapping;
+
+    /**
+     * Creates the clustering adapter. Only submissions that appear in those similarities might also appear in
+     * {@link ClusteringResult}s obtained from this adapter.
+     * @param comparisons that should be included in the process of clustering
+     * @param metric function that assigns a similarity to each comparison
+     */
+    public ClusteringAdapter(Collection<JPlagComparison> comparisons, Function<JPlagComparison, Float> metric) {
+        mapping = new IntegerMapping<>(comparisons.size());
+        for (JPlagComparison comparison : comparisons) {
+            mapping.map(comparison.getFirstSubmission());
+            mapping.map(comparison.getSecondSubmission());
+        }
+        int size = mapping.size();
+
+        similarityMatrix = new Array2DRowRealMatrix(size, size);
+        for (JPlagComparison comparison : comparisons) {
+            int firstIndex = mapping.map(comparison.getFirstSubmission());
+            int secondIndex = mapping.map(comparison.getSecondSubmission());
+            float similarity = metric.apply(comparison);
+            similarityMatrix.setEntry(firstIndex, secondIndex, similarity);
+            similarityMatrix.setEntry(secondIndex, firstIndex, similarity);
+        }
+    }
+
+    /**
+     * Use a generic clustering algorithm to cluster the submissions, that were included in this {@link ClusteringAdapter}'s
+     * comparison.
+     * @param algorithm that is used for clustering
+     * @return the clustered submissions
+     */
+    public ClusteringResult<Submission> doClustering(GenericClusteringAlgorithm algorithm) {
+        Collection<Collection<Integer>> intResult = algorithm.cluster(similarityMatrix);
+        ClusteringResult<Integer> modularityClusterResult = ClusteringResult.fromIntegerCollections(new ArrayList<>(intResult), similarityMatrix);
+        List<Cluster<Submission>> mappedClusters = modularityClusterResult.getClusters().stream().map(unmappedCluster -> {
+            return new Cluster<Submission>(unmappedCluster.getMembers().stream().map(mapping::unmap).collect(Collectors.toList()),
+                    unmappedCluster.getCommunityStrength());
+        }).collect(Collectors.toList());
+        return new ClusteringResult<>(mappedClusters, modularityClusterResult.getCommunityStrength());
+    }
+
+}

jplag/src/main/java/de/jplag/clustering/ClusteringAlgorithm.java

@@ -0,0 +1,14 @@
+package de.jplag.clustering;
+
+import de.jplag.clustering.algorithm.AgglomerativeClustering;
+import de.jplag.clustering.algorithm.SpectralClustering;
+
+/**
+ * Choosable clustering algorithms
+ */
+public enum ClusteringAlgorithm {
+    /** {@link AgglomerativeClustering} */
+    AGGLOMERATIVE,
+    /** {@link SpectralClustering} */
+    SPECTRAL
+}

jplag/src/main/java/de/jplag/clustering/ClusteringFactory.java

@@ -0,0 +1,56 @@
+package de.jplag.clustering;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import de.jplag.JPlagComparison;
+import de.jplag.Submission;
+import de.jplag.clustering.algorithm.AgglomerativeClustering;
+import de.jplag.clustering.algorithm.GenericClusteringAlgorithm;
+import de.jplag.clustering.algorithm.SpectralClustering;
+
+/**
+ * Runs the clustering according to an options object.
+ */
+public class ClusteringFactory {
+    public static List<ClusteringResult<Submission>> getClusterings(Collection<JPlagComparison> comparisons, ClusteringOptions options) {
+        if (!options.isEnabled())
+            return Collections.emptyList();
+
+        // init algorithm
+        GenericClusteringAlgorithm clusteringAlgorithm = null;
+        if (options.getAlgorithm() == ClusteringAlgorithm.AGGLOMERATIVE) {
+            clusteringAlgorithm = new AgglomerativeClustering(options);
+        } else if (options.getAlgorithm() == ClusteringAlgorithm.SPECTRAL) {
+            clusteringAlgorithm = new SpectralClustering(options);
+        }
+
+        // init preprocessor
+        Optional<ClusteringPreprocessor> preprocessor = options.getPreprocessor().constructPreprocessor(options);
+
+        if (preprocessor.isPresent()) {
+            // Package preprocessor into a clustering algorithm
+            clusteringAlgorithm = new PreprocessedClusteringAlgorithm(clusteringAlgorithm, preprocessor.orElseThrow());
+        }
+
+        // init adapter
+        ClusteringAdapter adapter = new ClusteringAdapter(comparisons, options.getSimilarityMetric());
+
+        // run clustering
+        ClusteringResult<Submission> result = adapter.doClustering(clusteringAlgorithm);
+
+        // remove bad clusters
+        result = removeBadClusters(result);
+
+        return List.of(result);
+    }
+
+    private static ClusteringResult<Submission> removeBadClusters(final ClusteringResult<Submission> clustering) {
+        List<Cluster<Submission>> filtered = clustering.getClusters().stream().filter(cluster -> !cluster.isBadCluster())
+                .collect(Collectors.toList());
+        return new ClusteringResult<>(filtered, clustering.getCommunityStrength());
+    }
+}

jplag/src/main/java/de/jplag/clustering/ClusteringOptions.java

@@ -0,0 +1,253 @@
+
+package de.jplag.clustering;
+
+import de.jplag.clustering.algorithm.InterClusterSimilarity;
+import de.jplag.options.SimilarityMetric;
+
+/**
+ * Collection of all possible parameters that describe how a clustering should be performed.
+ */
+public class ClusteringOptions {
+
+    public static final ClusteringOptions DEFAULTS = new Builder().build();
+
+    private final SimilarityMetric similarityMetric;
+    private final float spectralKernelBandwidth;
+    private final float spectralGaussianProcessVariance;
+    private final int spectralMinRuns;
+    private final int spectralMaxRuns;
+    private final int spectralMaxKMeansIterationPerRun;
+    private final float agglomerativeThreshold;
+    private final Preprocessing preprocessor;
+    private final boolean enabled;
+    private final ClusteringAlgorithm algorithm;
+    private final InterClusterSimilarity agglomerativeInterClusterSimilarity;
+    private final float preprocessorThreshold;
+    private final float preprocessorPercentile;
+
+    /**
+     * @return The similarity metric is used for clustering
+     */
+    public SimilarityMetric getSimilarityMetric() {
+        return similarityMetric;
+    }
+
+    /**
+     * The kernel bandwidth for the matern kernel used in the gaussian process for the automatic search for the number of
+     * clusters in spectral clustering. Affects the runtime and results of the spectral clustering.
+     * @return kernel bandwidth for spectral clustering
+     */
+    public float getSpectralKernelBandwidth() {
+        return spectralKernelBandwidth;
+    }
+
+    /**
+     * This is the assumed level of noise in the evaluation results of a spectral clustering. Acts as normalization
+     * parameter for the Gaussian Process. The default setting works well with similarity scores in the range between zero
+     * and one. Affects the results of the spectral clustering.
+     * @return assumed variance of noise in Gaussian Process during spectral clustering.
+     */
+    public float getSpectralGaussianProcessVariance() {
+        return spectralGaussianProcessVariance;
+    }
+
+    /**
+     * The minimal number of times the kMeans algorithm is run for the spectral clustering. These runs will use predefined
+     * numbers of clusters and will not use the bayesian optimization to determine the number of clusters.
+     * @return minimal kMeans runs during spectral clustering
+     */
+    public int getSpectralMinRuns() {
+        return spectralMinRuns;
+    }
+
+    /**
+     * The maximal number of times the kMeans algorithm is run during the bayesian optimization for spectral clustering. The
+     * bayesian optimization may be stopped before, when no more maxima of the acquisition-function are found.
+     * @return maximal kMeans runs during spectral clustering
+     */
+    public int getSpectralMaxRuns() {
+        return spectralMaxRuns;
+    }
+
+    /**
+     * Maximum number of iterations of the kMeans clustering per run during spectral clustering.
+     * @return maximal kMeans iterations
+     */
+    public int getSpectralMaxKMeansIterationPerRun() {
+        return spectralMaxKMeansIterationPerRun;
+    }
+
+    /**
+     * Agglomerative clustering will merge clusters that have a similarity higher than this threshold.
+     * @return merging threshold for agglomerative clustering
+     */
+    public float getAgglomerativeThreshold() {
+        return agglomerativeThreshold;
+    }
+
+    /**
+     * Preprocessing for the similarity values before clustering. Preprocessing is mandatory for spectral clustering and
+     * optional for agglomerative clustering.
+     * @return preprocessor
+     */
+    public Preprocessing getPreprocessor() {
+        return preprocessor;
+    }
+
+    /**
+     * @return whether clustering should be performed
+     */
+    public boolean isEnabled() {
+        return enabled;
+    }
+
+    /**
+     * @return the clustering algorithm to use
+     */
+    public ClusteringAlgorithm getAlgorithm() {
+        return algorithm;
+    }
+
+    /**
+     * Similarity measure between clusters in agglomerative clustering.
+     * @return similarity measure
+     */
+    public InterClusterSimilarity getAgglomerativeInterClusterSimilarity() {
+        return agglomerativeInterClusterSimilarity;
+    }
+
+    /**
+     * @return up to which similarity the threshold-preprocessor zeroes out the similarities
+     */
+    public float getPreprocessorThreshold() {
+        return preprocessorThreshold;
+    }
+
+    /**
+     * @return up to which percentile of similarities the percentile-preprocessor zeroes out the similarities
+     */
+    public float getPreprocessorPercentile() {
+        return preprocessorPercentile;
+    }
+
+    public static class Builder {
+
+        private SimilarityMetric similarityMetric;
+        private float spectralKernelBandwidth;
+        private float spectralGaussianProcessVariance;
+        private int spectralMinRuns;
+        private int spectralMaxRuns;
+        private int spectralMaxKMeansIterationPerRun;
+        private float agglomerativeThreshold;
+        private Preprocessing preprocessor;
+        private boolean enabled;
+        private ClusteringAlgorithm algorithm;
+        private InterClusterSimilarity agglomerativeInterClusterSimilarity;
+        private float preprocessorThreshold;
+        private float preprocessorPercentile;
+
+        public Builder() {
+            // Setting the defaults here
+            similarityMetric(SimilarityMetric.MAX);
+            spectralKernelBandwidth(20.f);
+            spectralGaussianProcessVariance(0.05f * 0.05f);
+            spectralMinRuns(5);
+            spectralMaxRuns(50);
+            spectralMaxKMeansIterationPerRun(200);
+            agglomerativeThreshold(0.2f);
+            preprocessor(Preprocessing.CUMULATIVE_DISTRIBUTION_FUNCTION);
+            enabled(true);
+            algorithm(ClusteringAlgorithm.SPECTRAL);
+            agglomerativeInterClusterSimilarity(InterClusterSimilarity.AVERAGE);
+            preprocessorThreshold(0.2f);
+            preprocessorPercentile(0.5f);
+        }
+
+        public Builder similarityMetric(SimilarityMetric similarityMetric) {
+            this.similarityMetric = similarityMetric;
+            return Builder.this;
+        }
+
+        public Builder spectralKernelBandwidth(float spectralKernelBandwidth) {
+            this.spectralKernelBandwidth = spectralKernelBandwidth;
+            return Builder.this;
+        }
+
+        public Builder spectralGaussianProcessVariance(float spectralGPVariance) {
+            this.spectralGaussianProcessVariance = spectralGPVariance;
+            return Builder.this;
+        }
+
+        public Builder spectralMinRuns(int spectralMinRuns) {
+            this.spectralMinRuns = spectralMinRuns;
+            return Builder.this;
+        }
+
+        public Builder spectralMaxRuns(int spectralMaxRuns) {
+            this.spectralMaxRuns = spectralMaxRuns;
+            return Builder.this;
+        }
+
+        public Builder spectralMaxKMeansIterationPerRun(int spectralMaxKMeansIterationPerRun) {
+            this.spectralMaxKMeansIterationPerRun = spectralMaxKMeansIterationPerRun;
+            return Builder.this;
+        }
+
+        public Builder agglomerativeThreshold(float agglomerativeThreshold) {
+            this.agglomerativeThreshold = agglomerativeThreshold;
+            return Builder.this;
+        }
+
+        public Builder preprocessor(Preprocessing preprocessor) {
+            this.preprocessor = preprocessor;
+            return Builder.this;
+        }
+
+        public Builder enabled(boolean enabled) {
+            this.enabled = enabled;
+            return Builder.this;
+        }
+
+        public Builder algorithm(ClusteringAlgorithm algorithm) {
+            this.algorithm = algorithm;
+            return Builder.this;
+        }
+
+        public Builder agglomerativeInterClusterSimilarity(InterClusterSimilarity agglomerativeInterClusterSimilarity) {
+            this.agglomerativeInterClusterSimilarity = agglomerativeInterClusterSimilarity;
+            return Builder.this;
+        }
+
+        public Builder preprocessorThreshold(float preprocessorThreshold) {
+            this.preprocessorThreshold = preprocessorThreshold;
+            return Builder.this;
+        }
+
+        public Builder preprocessorPercentile(float preprocessorPercentile) {
+            this.preprocessorPercentile = preprocessorPercentile;
+            return Builder.this;
+        }
+
+        public ClusteringOptions build() {
+
+            return new ClusteringOptions(this);
+        }
+    }
+
+    private ClusteringOptions(Builder builder) {
+        this.similarityMetric = builder.similarityMetric;
+        this.spectralKernelBandwidth = builder.spectralKernelBandwidth;
+        this.spectralGaussianProcessVariance = builder.spectralGaussianProcessVariance;
+        this.spectralMinRuns = builder.spectralMinRuns;
+        this.spectralMaxRuns = builder.spectralMaxRuns;
+        this.spectralMaxKMeansIterationPerRun = builder.spectralMaxKMeansIterationPerRun;
+        this.agglomerativeThreshold = builder.agglomerativeThreshold;
+        this.preprocessor = builder.preprocessor;
+        this.enabled = builder.enabled;
+        this.algorithm = builder.algorithm;
+        this.agglomerativeInterClusterSimilarity = builder.agglomerativeInterClusterSimilarity;
+        this.preprocessorThreshold = builder.preprocessorThreshold;
+        this.preprocessorPercentile = builder.preprocessorPercentile;
+    }
+
+}

jplag/src/main/java/de/jplag/clustering/ClusteringPreprocessor.java

@@ -0,0 +1,23 @@
+package de.jplag.clustering;
+
+/**
+ * Interface for classes that process similarity matrices before any clustering. Classes implementing this interface
+ * must ensure that they do not produce zero rows/columns inside the similarity matrix. They must also be able to
+ * calculate the original indices of rows/columns after use through their {@link ClusteringPreprocessor#originalIndexOf}
+ * method.
+ */
+public interface ClusteringPreprocessor {
+    /**
+     * Applies some preprocessing defined by the implementing class.
+     * @param similarityMatrix original similarities
+     * @return preprocessed similarities
+     */
+    double[][] preprocessSimilarities(double[][] similarityMatrix);
+
+    /**
+     * Maps the indices of the preprocessed similarity matrix back to indices in the original matrix.
+     * @param index after preprocessing
+     * @return corresponding index before preprocessing
+     */
+    int originalIndexOf(int index);
+}

jplag/src/main/java/de/jplag/clustering/ClusteringResult.java

@@ -0,0 +1,102 @@
+package de.jplag.clustering;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiFunction;
+import java.util.stream.DoubleStream;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.linear.RealMatrix;
+
+/**
+ * Set of clusters dividing a set of entities.
+ * @param <T> type of the clustered entities (e.g. Submission)
+ */
+public class ClusteringResult<T> {
+
+    private final List<Cluster<T>> clusters;
+    private float communityStrength = 0;
+
+    public ClusteringResult(Collection<Cluster<T>> clusters, float communityStrength) {
+        this.clusters = Collections.unmodifiableList(new ArrayList<>(clusters));
+        this.communityStrength = communityStrength;
+        for (Cluster<T> cluster : clusters) {
+            cluster.setClusteringResult(this);
+        }
+    }
+
+    public Collection<Cluster<T>> getClusters() {
+        return Collections.unmodifiableList(clusters);
+    }
+
+    /**
+     * Community strength of the clustering. The expectation of community strength in a random graph is zero and it can not
+     * exceed one. It's the sum of it's clusters {@link Cluster#getCommunityStrength}. If the underlying network is not
+     * changed, a higher community strength denotes a better clustering. See: Finding and evaluating community structure in
+     * networks, M. E. J. Newman and M. Girvan, Phys. Rev. E 69, 026113 – Published 26 February 2004, Doi:
+     * 10.1103/PhysRevE.69.026113 It's called modularity in that paper.
+     * @return community strength
+     */
+    public float getCommunityStrength() {
+        return communityStrength;
+    }
+
+    /**
+     * How much this clustering result is worth during optimization.
+     * @param similarity
+     * @return worth
+     */
+    public float getWorth(BiFunction<T, T, Float> similarity) {
+        return (float) getClusters().stream().mapToDouble(c -> c.getWorth(similarity)).map(worth -> Double.isFinite(worth) ? worth : 0).average()
+                .getAsDouble();
+    }
+
+    /**
+     * Responsible for calculating the {@link ClusteringResult#getCommunityStrength} of a new clustering on integers and
+     * it's clusters.
+     */
+    public static ClusteringResult<Integer> fromIntegerCollections(List<Collection<Integer>> clustering, RealMatrix similarity) {
+        int numberOfSubmissions = similarity.getRowDimension();
+        Map<Integer, Integer> clusterIndicesOfSubmissionIndices = new HashMap<>();
+        int clusterIdx = 0;
+        for (Collection<Integer> cluster : clustering) {
+            for (Integer submissionIdx : cluster) {
+                clusterIndicesOfSubmissionIndices.put(submissionIdx, clusterIdx);
+            }
+            clusterIdx++;
+        }
+        List<Cluster<Integer>> clusters = new ArrayList<>(clustering.size());
+        float communityStrength = 0;
+        if (clustering.size() > 0) {
+            RealMatrix percentagesOfSimilaritySums = new Array2DRowRealMatrix(clustering.size(), clustering.size());
+            percentagesOfSimilaritySums = percentagesOfSimilaritySums.scalarMultiply(0);
+            for (int i = 0; i < numberOfSubmissions; i++) {
+                if (!clusterIndicesOfSubmissionIndices.containsKey(i))
+                    continue;
+                int clusterA = clusterIndicesOfSubmissionIndices.get(i);
+                for (int j = i + 1; j < numberOfSubmissions; j++) {
+                    if (!clusterIndicesOfSubmissionIndices.containsKey(j))
+                        continue;
+                    int clusterB = clusterIndicesOfSubmissionIndices.get(j);
+                    percentagesOfSimilaritySums.addToEntry(clusterA, clusterB, similarity.getEntry(i, j));
+                    percentagesOfSimilaritySums.addToEntry(clusterB, clusterA, similarity.getEntry(i, j));
+                }
+            }
+            percentagesOfSimilaritySums = percentagesOfSimilaritySums
+                    .scalarMultiply(1 / Arrays.stream(similarity.getData()).flatMapToDouble(DoubleStream::of).sum());
+            for (int i = 0; i < clustering.size(); i++) {
+                double outWeightSum = percentagesOfSimilaritySums.getRowVector(i).getL1Norm();
+                double clusterCommunityStrength = percentagesOfSimilaritySums.getEntry(i, i) - outWeightSum * outWeightSum;
+                clusters.add(new Cluster<Integer>(clustering.get(i), (float) clusterCommunityStrength));
+                communityStrength += clusterCommunityStrength;
+            }
+        }
+        return new ClusteringResult<>(clusters, communityStrength);
+    }
+
+}

jplag/src/main/java/de/jplag/clustering/IntegerMapping.java

@@ -0,0 +1,51 @@
+package de.jplag.clustering;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * Creates a mapping of any set to integers
+ */
+public class IntegerMapping<T> {
+
+    private HashMap<T, Integer> mapping = new HashMap<>();
+    private List<T> backMapping = new ArrayList<>();
+    private int size = 0;
+
+    public IntegerMapping(int initialCapacity) {
+        mapping = new HashMap<>(initialCapacity);
+        backMapping = new ArrayList<>(initialCapacity);
+    }
+
+    /**
+     * @param value is added to the mapping (if not already present)
+     * @return the associated integer
+     */
+    public int map(T value) {
+        Integer result = mapping.get(value);
+        if (result == null) {
+            int newIndex = size++;
+            mapping.put(value, newIndex);
+            backMapping.add(value);
+            return newIndex;
+        }
+        return result;
+    }
+
+    /**
+     * Maps the integer back to the original set.
+     * @param index the integer
+     * @return the original value
+     */
+    public T unmap(int index) {
+        return backMapping.get(index);
+    }
+
+    /**
+     * @return Number of unique values in the mapping
+     */
+    public int size() {
+        return size;
+    }
+}

jplag/src/main/java/de/jplag/clustering/PreprocessedClusteringAlgorithm.java

@@ -0,0 +1,36 @@
+package de.jplag.clustering;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.stream.Collectors;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.linear.RealMatrix;
+
+import de.jplag.clustering.algorithm.GenericClusteringAlgorithm;
+
+/**
+ * Adapter class to put a preprocessor before any clustering algorithm.
+ */
+public class PreprocessedClusteringAlgorithm implements GenericClusteringAlgorithm {
+
+    private final GenericClusteringAlgorithm base;
+    private final ClusteringPreprocessor preprocessor;
+
+    public PreprocessedClusteringAlgorithm(GenericClusteringAlgorithm base, ClusteringPreprocessor preprocessor) {
+        this.base = base;
+        this.preprocessor = preprocessor;
+    }
+
+    @Override
+    public Collection<Collection<Integer>> cluster(RealMatrix similarityMatrix) {
+        double[][] data = preprocessor.preprocessSimilarities(similarityMatrix.getData());
+        if (data.length > 2) {
+            Collection<Collection<Integer>> preliminaryResult = base.cluster(new Array2DRowRealMatrix(data, false));
+            return preliminaryResult.stream().map(cluster -> cluster.stream().map(preprocessor::originalIndexOf).collect(Collectors.toList()))
+                    .collect(Collectors.toList());
+        }
+        return Collections.emptyList();
+    }
+
+}

jplag/src/main/java/de/jplag/clustering/Preprocessing.java

@@ -0,0 +1,33 @@
+package de.jplag.clustering;
+
+import java.util.Optional;
+import java.util.function.Function;
+
+import antlr.preprocessor.Preprocessor;
+
+import de.jplag.clustering.preprocessors.CumulativeDistributionFunctionPreprocessor;
+import de.jplag.clustering.preprocessors.PercentileThresholdProcessor;
+import de.jplag.clustering.preprocessors.ThresholdPreprocessor;
+
+/**
+ * List of all usable {@link Preprocessor}s.
+ */
+public enum Preprocessing {
+    NONE(options -> null),
+    /** {@link CumulativeDistributionFunctionPreprocessor} */
+    CUMULATIVE_DISTRIBUTION_FUNCTION(options -> new CumulativeDistributionFunctionPreprocessor()),
+    /** {@link ThresholdPreprocessor} */
+    THRESHOLD(options -> new ThresholdPreprocessor(options.getPreprocessorThreshold())),
+    /** {@link PercentileThresholdProcessor} */
+    PERCENTILE(options -> new PercentileThresholdProcessor(options.getPreprocessorPercentile()));
+
+    private final Function<ClusteringOptions, ClusteringPreprocessor> constructor;
+
+    private Preprocessing(Function<ClusteringOptions, ClusteringPreprocessor> constructor) {
+        this.constructor = constructor;
+    }
+
+    public Optional<ClusteringPreprocessor> constructPreprocessor(ClusteringOptions options) {
+        return Optional.ofNullable(constructor.apply(options));
+    }
+}

jplag/src/main/java/de/jplag/clustering/PreprocessorHelper.java

@@ -0,0 +1,48 @@
+package de.jplag.clustering;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.DoubleStream;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.linear.RealMatrix;
+
+/**
+ * Implements removal of zero rows / columns and calculation of original indices for classes implementing
+ * {@link ClusteringPreprocessor}.
+ */
+public class PreprocessorHelper {
+    private IntegerMapping<Integer> mapping;
+
+    /**
+     * Removes disconnected edges from the input matrix.
+     * @param connections similarity matrix
+     * @return similarity matrix without zero rows / cols
+     */
+    public double[][] removeDisconnectedEntries(double[][] connections) {
+        List<Integer> rowList = new ArrayList<>();
+        mapping = new IntegerMapping<>(connections.length);
+        RealMatrix similarity = new Array2DRowRealMatrix(connections, true);
+        for (int i = 0; i < similarity.getRowDimension(); i++) {
+            if (DoubleStream.of(similarity.getRow(i)).filter(x -> x > 0).findAny().isPresent()) {
+                rowList.add(i);
+                mapping.map(i);
+            }
+        }
+        int[] preservedRows = rowList.stream().mapToInt(Integer::intValue).toArray();
+        if (preservedRows.length == 0) {
+            return new double[0][];
+        }
+        similarity = similarity.getSubMatrix(preservedRows, preservedRows);
+        return similarity.getData();
+    }
+
+    /**
+     * Implements the logic for {@link ClusteringPreprocessor#originalIndexOf}
+     * @param index after preprocessing
+     * @return corresponding index before preprocessing
+     */
+    public int postProcessResult(int index) {
+        return mapping.unmap(index);
+    }
+}

jplag/src/main/java/de/jplag/clustering/algorithm/AgglomerativeClustering.java

@@ -0,0 +1,120 @@
+package de.jplag.clustering.algorithm;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.math3.linear.RealMatrix;
+
+import de.jplag.clustering.ClusteringOptions;
+
+/**
+ * Begin by assigning a cluster to each entity and then successively merge similar clusters.
+ */
+public class AgglomerativeClustering implements GenericClusteringAlgorithm {
+
+    private ClusteringOptions options;
+
+    public AgglomerativeClustering(ClusteringOptions options) {
+        this.options = options;
+    }
+
+    @Override
+    public Collection<Collection<Integer>> cluster(RealMatrix similarityMatrix) {
+        int size = similarityMatrix.getRowDimension();
+        // all clusters that do not have a parent yet
+        Set<Cluster> clusters = new HashSet<>(size);
+        // calculated similarities. Might contain connections to already visited
+        // clusters, those need to be ignored.
+        PriorityQueue<ClusterConnection> similarities;
+
+        {
+            // initialization
+
+            List<Cluster> initialClusters = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                List<Integer> members = new ArrayList<>();
+                members.add(i);
+                Cluster cluster = new Cluster(members);
+                initialClusters.add(cluster);
+                clusters.add(cluster);
+            }
+
+            List<ClusterConnection> initialSimilarities = new ArrayList<>(size * (size - 1) / 2);
+
+            for (int leftIndex = 0; leftIndex < initialClusters.size(); leftIndex++) {
+                Cluster leftCluster = initialClusters.get(leftIndex);
+                for (int rightIndex = leftIndex + 1; rightIndex < initialClusters.size(); rightIndex++) {
+                    Cluster rightCluster = initialClusters.get(rightIndex);
+                    initialSimilarities
+                            .add(new ClusterConnection(leftCluster, rightCluster, (float) similarityMatrix.getEntry(leftIndex, rightIndex)));
+                }
+            }
+
+            similarities = new PriorityQueue<>(initialSimilarities);
+        }
+
+        while (clusters.size() > 1) {
+            ClusterConnection nearest = similarities.poll();
+            if (!(clusters.contains(nearest.left) && clusters.contains(nearest.right))) {
+                // One cluster already part of another cluster
+                continue;
+            }
+            if (nearest.similarity < options.getAgglomerativeThreshold()) {
+                break;
+            }
+            clusters.remove(nearest.left);
+            clusters.remove(nearest.right);
+            nearest.left.getSubmissions().addAll(nearest.right.getSubmissions());
+            Cluster combined = new Cluster(nearest.left.getSubmissions());
+            for (Cluster otherCluster : clusters) {
+                float similarity = options.getAgglomerativeInterClusterSimilarity().clusterSimilarity(combined.submissions, otherCluster.submissions,
+                        similarityMatrix);
+                similarities.add(new ClusterConnection(combined, otherCluster, similarity));
+            }
+            clusters.add(combined);
+        }
+
+        return clusters.stream().map(Cluster::getSubmissions).collect(Collectors.toList());
+    }
+
+    private class ClusterConnection implements Comparable<ClusterConnection> {
+
+        private Cluster left;
+        private Cluster right;
+        private float similarity;
+
+        public ClusterConnection(Cluster left, Cluster right, float similarity) {
+            this.left = left;
+            this.right = right;
+            this.similarity = similarity;
+        }
+
+        @Override
+        public int compareTo(ClusterConnection other) {
+            return (int) Math.signum(other.similarity - similarity);
+        }
+
+    }
+
+    /**
+     * Encapsulate a list in a class because we do not want hashing based on the members but on identity only. Also a
+     * cluster need a different identity than the list because the lists are reused.
+     */
+    private class Cluster {
+        private List<Integer> submissions;
+
+        public Cluster(List<Integer> submissions) {
+            this.submissions = submissions;
+        }
+
+        public List<Integer> getSubmissions() {
+            return submissions;
+        }
+    }
+
+}

jplag/src/main/java/de/jplag/clustering/algorithm/BFGS.java

@@ -0,0 +1,835 @@
+/**
+ * The original version of this file originates from SMILE: https://github.com/haifengl/smile
+ */
+
+/*******************************************************************************
+ * Copyright (c) 2010-2020 Haifeng Li. All rights reserved. Smile is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation,
+ * either version 3 of the License, or (at your option) any later version. Smile is distributed in the hope that it will
+ * be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+ * PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU
+ * Lesser General Public License along with Smile. If not, see <https://www.gnu.org/licenses/>.
+ ******************************************************************************/
+
+package de.jplag.clustering.algorithm;
+
+import static java.lang.Math.abs;
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static java.lang.Math.sqrt;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.linear.ArrayRealVector;
+import org.apache.commons.math3.linear.DecompositionSolver;
+import org.apache.commons.math3.linear.LUDecomposition;
+import org.apache.commons.math3.linear.MatrixUtils;
+import org.apache.commons.math3.linear.RealMatrix;
+import org.apache.commons.math3.util.Pair;
+
+/**
+ * The Broyden–Fletcher–Goldfarb–Shanno (BFGS) algorithm is an iterative method for solving unconstrained nonlinear
+ * optimization problems.
+ * <p>
+ * The BFGS method belongs to quasi-Newton methods, a class of hill-climbing optimization techniques that seek a
+ * stationary point of a (preferably twice continuously differentiable) function. For such problems, a necessary
+ * condition for optimality is that the gradient be zero. Newton's method and the BFGS methods are not guaranteed to
+ * converge unless the function has a quadratic Taylor expansion near an optimum. However, BFGS has proven to have good
+ * performance even for non-smooth optimizations.
+ * <p>
+ * In quasi-Newton methods, the Hessian matrix of second derivatives is not computed. Instead, the Hessian matrix is
+ * approximated using updates specified by gradient evaluations (or approximate gradient evaluations). Quasi-Newton
+ * methods are generalizations of the secant method to find the root of the first derivative for multidimensional
+ * problems. In multi-dimensional problems, the secant equation does not specify a unique solution, and quasi-Newton
+ * methods differ in how they constrain the solution. The BFGS method is one of the most popular members of this class.
+ * <p>
+ * Like the original BFGS, the limited-memory BFGS (L-BFGS) uses an estimation to the inverse Hessian matrix to steer
+ * its search through variable space, but where BFGS stores a dense <code>n × n</code> approximation to the inverse
+ * Hessian (<code>n</code> being the number of variables in the problem), L-BFGS stores only a few vectors that
+ * represent the approximation implicitly. Due to its resulting linear memory requirement, the L-BFGS method is
+ * particularly well suited for optimization problems with a large number of variables (e.g., &gt; 1000). Instead of the
+ * inverse Hessian <code>H_k</code>, L-BFGS maintains * a history of the past <code>m</code> updates of the position
+ * <code>x</code> and gradient <code>∇f(x)</code>, where generally the history size <code>m</code> can be small (often
+ * <code>m &lt; 10</code>). These updates are used to implicitly do operations requiring the <code>H_k</code>-vector
+ * product.
+ * <h2>References</h2>
+ * <ol>
+ * <li>Roger Fletcher. Practical methods of optimization.</li>
+ * <li>D. Liu and J. Nocedal. On the limited memory BFGS method for large scale optimization. Mathematical Programming B
+ * 45 (1989) 503-528.</li>
+ * <li>Richard H. Byrd, Peihuang Lu, Jorge Nocedal and Ciyou Zhu. A limited memory algorithm for bound constrained
+ * optimization.</li>
+ * </ol>
+ * @author Haifeng Li
+ */
+public class BFGS {
+    // private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BFGS.class);
+    /** A number close to zero, between machine epsilon and its square root. */
+    private static final double EPSILON = Double.valueOf(System.getProperty("smile.bfgs.epsilon", "1E-10"));
+    /** The convergence criterion on function value. */
+    private static final double TOLF = 4 * EPSILON;
+    /** The scaled maximum step length allowed in line searches. */
+    private static final double STPMX = 100.0;
+
+    /**
+     * Minimize a function along a search direction by find a step which satisfies a sufficient decrease condition and a
+     * curvature condition.
+     * <p>
+     * At each stage this function updates an interval of uncertainty with endpoints <code>stx</code> and <code>sty</code>.
+     * The interval of uncertainty is initially chosen so that it contains a minimizer of the modified function
+     * <p>
+     * 
+     * <pre>
+     *      f(x+stp*s) - f(x) - ftol*stp*(gradf(x)'s).
+     * </pre>
+     * 
+     * If a step is obtained for which the modified function has a nonpositive function value and nonnegative derivative,
+     * then the interval of uncertainty is chosen so that it contains a minimizer of <code>f(x+stp*s)</code>.
+     * <p>
+     * The algorithm is designed to find a step which satisfies the sufficient decrease condition
+     * <p>
+     * 
+     * <pre>
+     *       f(x+stp*s) &lt;= f(X) + ftol*stp*(gradf(x)'s),
+     * </pre>
+     * 
+     * and the curvature condition
+     * <p>
+     * 
+     * <pre>
+     *       abs(gradf(x+stp*s)'s)) &lt;= gtol*abs(gradf(x)'s).
+     * </pre>
+     * 
+     * If <code>ftol</code> is less than <code>gtol</code> and if, for example, the function is bounded below, then there is
+     * always a step which satisfies both conditions. If no step can be found which satisfies both conditions, then the
+     * algorithm usually stops when rounding errors prevent further progress. In this case <code>stp</code> only satisfies
+     * the sufficient decrease condition.
+     * @param xold on input this contains the base point for the line search.
+     * @param fold on input this contains the value of the objective function at <code>x</code>.
+     * @param g on input this contains the gradient of the objective function at <code>x</code>.
+     * @param p the search direction.
+     * @param x on output, it contains <code>xold + &gamma;*p</code>, where &gamma; > 0 is the step length.
+     * @param stpmax specify upper bound for the step in the line search so that we do not try to evaluate the function in
+     * regions where it is undefined or subject to overflow.
+     * @return the new function value.
+     */
+    private static double linesearch(MultivariateFunction func, double[] xold, double fold, double[] g, double[] p, double[] x, double stpmax) {
+        if (stpmax <= 0) {
+            throw new IllegalArgumentException("Invalid upper bound of linear search step: " + stpmax);
+        }
+
+        // Termination occurs when the relative width of the interval
+        // of uncertainty is at most xtol.
+        final double xtol = EPSILON;
+        // Tolerance for the sufficient decrease condition.
+        final double ftol = 1.0E-4;
+
+        int n = xold.length;
+
+        // Scale if attempted step is too big
+        double pnorm = norm(p);
+        if (pnorm > stpmax) {
+            double r = stpmax / pnorm;
+            for (int i = 0; i < n; i++) {
+                p[i] *= r;
+            }
+        }
+
+        // Check if s is a descent direction.
+        double slope = 0.0;
+        for (int i = 0; i < n; i++) {
+            slope += g[i] * p[i];
+        }
+
+        if (slope >= 0) {
+            // throw new IllegalArgumentException("Line Search: the search direction is not a descent direction, which may be caused
+            // by roundoff problem.");
+        }
+
+        // Calculate minimum step.
+        double test = 0.0;
+        for (int i = 0; i < n; i++) {
+            double temp = abs(p[i]) / max(xold[i], 1.0);
+            if (temp > test) {
+                test = temp;
+            }
+        }
+
+        double alammin = xtol / test;
+        double alam = 1.0;
+
+        double alam2 = 0.0, f2 = 0.0;
+        double a, b, disc, rhs1, rhs2, tmpalam;
+        int runs = 50;
+        while (true) {
+            // Evaluate the function and gradient at stp
+            // and compute the directional derivative.
+            for (int i = 0; i < n; i++) {
+                x[i] = xold[i] + alam * p[i];
+            }
+
+            double f = func.f(x);
+
+            // Convergence on &Delta; x.
+            if (alam < alammin) {
+                System.arraycopy(xold, 0, x, 0, n);
+                return f;
+            } else if (f <= fold + ftol * alam * slope) {
+                // Sufficient function decrease.
+                return f;
+            } else if (runs-- < 0) {
+                return f;
+            } else {
+                // Backtrack
+                if (alam == 1.0) {
+                    // First time
+                    tmpalam = -slope / (2.0 * (f - fold - slope));
+                } else {
+                    // Subsequent backtracks.
+                    rhs1 = f - fold - alam * slope;
+                    rhs2 = f2 - fold - alam2 * slope;
+                    a = (rhs1 / (alam * alam) - rhs2 / (alam2 * alam2)) / (alam - alam2);
+                    b = (-alam2 * rhs1 / (alam * alam) + alam * rhs2 / (alam2 * alam2)) / (alam - alam2);
+                    if (a == 0.0) {
+                        tmpalam = -slope / (2.0 * b);
+                    } else {
+                        disc = b * b - 3.0 * a * slope;
+                        if (disc < 0.0) {
+                            tmpalam = 0.5 * alam;
+                        } else if (b <= 0.0) {
+                            tmpalam = (-b + sqrt(disc)) / (3.0 * a);
+                        } else {
+                            tmpalam = -slope / (b + sqrt(disc));
+                        }
+                    }
+                    if (tmpalam > 0.5 * alam) {
+                        tmpalam = 0.5 * alam;
+                    }
+                }
+            }
+            alam2 = alam;
+            f2 = f;
+            alam = max(tmpalam, 0.1 * alam);
+        }
+    }
+
+    /**
+     * This method solves the bound constrained minimization problem using the L-BFGS-B method. The L-BFGS-B algorithm
+     * extends L-BFGS to handle simple box constraints on variables; that is, constraints of the form li ≤ xi ≤ ui where li
+     * and ui are per-variable constant lower and upper bounds, respectively (for each xi, either or both bounds may be
+     * omitted). The method works by identifying fixed and free variables at every step (using a simple gradient method),
+     * and then using the L-BFGS method on the free variables only to get higher accuracy, and then repeating the process.
+     * @param func the function to be minimized.
+     * @param m the number of corrections used in the L-BFGS update. Values of <code>m</code> less than 3 are not
+     * recommended; large values of <code>m</code> will result in excessive computing time. <code>3 &lt;= m &lt;= 7</code>
+     * is recommended. A common choice for m is m = 5.
+     * @param x on initial entry this must be set by the user to the values of the initial estimate of the solution vector.
+     * On exit with <code>iflag = 0</code>, it contains the values of the variables at the best point found (usually a
+     * solution).
+     * @param l the lower bound.
+     * @param u the upper bound.
+     * @param gtol the convergence tolerance on zeroing the gradient.
+     * @param maxIter the maximum number of iterations.
+     * @return the minimum value of the function.
+     */
+    public static double minimize(DifferentiableMultivariateFunction func, int m, double[] x, double[] l, double[] u, double gtol, int maxIter) {
+        if (gtol <= 0.0) {
+            throw new IllegalArgumentException("Invalid gradient tolerance: " + gtol);
+        }
+
+        if (maxIter <= 0) {
+            throw new IllegalArgumentException("Invalid maximum number of iterations: " + maxIter);
+        }
+
+        if (m <= 0) {
+            throw new IllegalArgumentException("Invalid m: " + m);
+        }
+
+        if (l.length != x.length) {
+            throw new IllegalArgumentException("Invalid lower bound size: " + l.length);
+        }
+
+        if (u.length != x.length) {
+            throw new IllegalArgumentException("Invalid upper bound size: " + u.length);
+        }
+
+        int n = x.length;
+        double theta = 1.0;
+
+        Matrix Y = null, S = null;
+        Matrix W = new Matrix(n, 1);
+        Matrix M = new Matrix(1, 1);
+
+        ArrayList<double[]> yHistory = new ArrayList<>();
+        ArrayList<double[]> sHistory = new ArrayList<>();
+        double[] y = new double[n];
+        double[] s = new double[n];
+
+        double[] p = new double[n];
+        double[] g = new double[n];
+        double[] cauchy = new double[n];
+        double f = func.g(x, g);
+
+        double[] x_old = new double[n];
+        double[] g_old = new double[n];
+
+        // Upper limit for line search step.
+        double stpmax = STPMX * max(norm(x), n);
+
+        for (int iter = 1; iter <= maxIter; iter++) {
+            double f_old = f;
+            System.arraycopy(x, 0, x_old, 0, n);
+            System.arraycopy(g, 0, g_old, 0, n);
+
+            // STEP 2: compute the cauchy point
+            System.arraycopy(x, 0, cauchy, 0, n);
+            double[] c = cauchy(x, g, cauchy, l, u, theta, W, M);
+            clampToBound(cauchy, l, u);
+
+            // STEP 3: compute a search direction d_k by the primal method for the sub-problem
+            double[] subspaceMin = subspaceMinimization(x, g, cauchy, c, l, u, theta, W, M);
+            clampToBound(subspaceMin, l, u);
+
+            // STEP 4: perform line search
+            for (int i = 0; i < n; i++) {
+                p[i] = subspaceMin[i] - x[i];
+            }
+            linesearch(func, x_old, f_old, g, p, x, stpmax);
+            clampToBound(x, l, u);
+
+            for (double xi : x) {
+                if (Double.isNaN(xi) || Double.isInfinite(xi)) {
+                    // logger.warn("L-BFGS-B: bad x produced by line search, return previous good x");
+                    System.arraycopy(x_old, 0, x, 0, n);
+                    return f_old;
+                }
+            }
+
+            // STEP 5: compute gradient update current guess and function information
+            f = func.g(x, g);
+            if (Double.isNaN(f) || Double.isInfinite(f)) {
+                // logger.warn("L-BFGS-B: bad f(x) produced by line search, return previous good x");
+                System.arraycopy(x_old, 0, x, 0, n);
+                return f_old;
+            }
+
+            if (gnorm(x, g, l, u) < gtol) {
+                // logger.info(String.format("L-BFGS-B converges on gradient after %d iterations: %.5f", iter, f));
+                return f;
+            }
+
+            if (iter % 100 == 0) {
+                // logger.info(String.format("L-BFGS-B: the function value after %3d iterations: %.5f", iter, f));
+            }
+
+            // prepare for next iteration
+            for (int i = 0; i < n; i++) {
+                y[i] = g[i] - g_old[i];
+                s[i] = x[i] - x_old[i];
+            }
+
+            // STEP 6
+            double sy = dot(s, y);
+            double yy = dot(y, y);
+            double test = abs(sy);
+            if (test > EPSILON * yy) {
+                if (yHistory.size() >= m) {
+                    yHistory.remove(0);
+                    sHistory.remove(0);
+                }
+
+                yHistory.add(y);
+                sHistory.add(s);
+
+                int h = yHistory.size();
+                if (iter <= m) {
+                    Y = new Matrix(n, h);
+                    S = new Matrix(n, h);
+                    W = new Matrix(n, 2 * h);
+                    M = new Matrix(2 * h, 2 * h);
+                }
+
+                // STEP 7
+                theta = yy / sy;
+                for (int j = 0; j < h; j++) {
+                    double[] yj = yHistory.get(j);
+                    double[] sj = sHistory.get(j);
+
+                    for (int i = 0; i < n; i++) {
+                        Y.set(i, j, yj[i]);
+                        S.set(i, j, sj[i]);
+                        W.set(i, j, yj[i]);
+                        W.set(i, h + j, sj[i] * theta);
+                    }
+                }
+
+                Matrix SY = S.tm(Y);
+                Matrix SS = S.ata();
+                for (int j = 0; j < h; j++) {
+                    M.set(j, j, -SY.get(j, j));
+                    for (int i = 0; i <= j; i++) {
+                        M.set(h + i, j, 0.0);
+                        M.set(j, h + i, 0.0);
+                    }
+
+                    for (int i = j + 1; i < h; i++) {
+                        M.set(h + i, j, SY.get(i, j));
+                        M.set(j, h + i, SY.get(i, j));
+                    }
+
+                    for (int i = 0; i < h; i++) {
+                        M.set(h + i, h + j, theta * SS.get(i, j));
+                    }
+                }
+
+                // M.uplo(UPLO.LOWER);
+                M = M.inverse();
+            }
+
+            // logger.debug("L-BFGS-B iteration {} moves from {} to {} where f(x) = {}", iter, Arrays.toString(x_old),
+            // Arrays.toString(x), f);
+
+            if (abs(f_old - f) < TOLF) {
+                // logger.info(String.format("L-BFGS-B converges on f(x) after %d iterations: %.5f", iter, f));
+                return f;
+            }
+        }
+
+        // logger.warn(String.format("L-BFGS-B reaches maximum %d iterations: %.5f", maxIter, f));
+        return f;
+    }
+
+    private static double dot(double[] a, double[] b) {
+        if (a.length != b.length) {
+            throw new IllegalArgumentException("a and b have different lengths");
+        }
+        double sum = 0;
+        for (int i = 0; i < a.length; i++) {
+            sum += a[i] * b[i];
+        }
+        return sum;
+    }
+
+    private static double norm(double[] a) {
+        return Math.sqrt(dot(a, a));
+    }
+
+    /**
+     * Algorithm CP: Computation of the Generalized Cauchy Point. See page 8.
+     * @param x the parameter vector
+     * @param g the gradient vector
+     */
+    private static double[] cauchy(double[] x, double[] g, double[] cauchy, double[] l, double[] u, double theta, Matrix W, Matrix M) {
+        int n = x.length;
+        double[] t = new double[n];
+        double[] d = new double[n];
+
+        for (int i = 0; i < n; i++) {
+            t[i] = g[i] == 0 ? Double.MAX_VALUE : (g[i] < 0 ? (x[i] - u[i]) / g[i] : (x[i] - l[i]) / g[i]);
+            if (t[i] != 0.0)
+                d[i] = -g[i];
+        }
+
+        int[] index = sortWithIndex(t);
+        double[] p = W.tv(d);
+        double[] c = new double[p.length];
+        double fPrime = -dot(d, d);
+        double fDoublePrime = max(-theta * fPrime - M.xAx(p), EPSILON);
+        double f_dp_orig = fDoublePrime;
+        double dt_min = -fPrime / fDoublePrime;
+        double t_old = 0.0;
+
+        int i = 0;
+        for (; i < n; i++) {
+            if (t[index[i]] >= 0)
+                break;
+        }
+
+        double dt = i < n ? t[i] : 0;
+        for (; dt_min >= dt && i < n; i++) {
+            int b = index[i];
+            double tb = t[i];
+            dt = tb - t_old;
+
+            cauchy[b] = d[b] > 0 ? u[b] : (d[b] < 0 ? l[b] : cauchy[b]);
+            double zb = cauchy[b] - x[b];
+            for (int j = 0; j < c.length; j++) {
+                c[j] += p[j] * dt;
+            }
+
+            double gb = g[b];
+            double[] wbt = W.row(b);
+            fPrime += dt * fDoublePrime + gb * gb + theta * gb * zb - gb * dot(wbt, M.mv(c));
+            fDoublePrime -= theta * gb * gb + 2.0 * gb * dot(wbt, M.mv(p)) + gb * gb * M.xAx(wbt);
+            fDoublePrime = max(fDoublePrime, EPSILON * f_dp_orig);
+            for (int j = 0; j < p.length; j++) {
+                p[j] += wbt[j] * gb;
+            }
+
+            d[b] = 0;
+            dt_min = -fPrime / fDoublePrime;
+            t_old = tb;
+        }
+
+        dt_min = max(dt_min, 0.0);
+        t_old += dt_min;
+
+        for (int ii = i; ii < n; ii++) {
+            int si = index[ii];
+            cauchy[si] = x[si] + t_old * d[si];
+        }
+
+        for (int j = 0; j < c.length; j++) {
+            c[j] += p[j] * dt_min;
+        }
+
+        return c;
+    }
+
+    /**
+     * Minimization of the subspace of free variables. See Section 5 on page 9.
+     * @param x the parameter vector
+     * @param g the gradient vector
+     * @param cauchy the cauchy points
+     * @param c vector obtained from gcp() used to initialize the subspace minimization process
+     */
+    private static double[] subspaceMinimization(double[] x, double[] g, double[] cauchy, double[] c, double[] l, double[] u, double theta, Matrix W,
+            Matrix M) {
+        int n = x.length;
+        double thetaInverse = 1.0 / theta;
+        ArrayList<Integer> freeVarIdx = new ArrayList<>();
+        for (int i = 0; i < n; i++) {
+            if (cauchy[i] != u[i] && cauchy[i] != l[i]) {
+                freeVarIdx.add(i);
+            }
+        }
+
+        if (freeVarIdx.isEmpty()) {
+            return cauchy.clone();
+        }
+
+        int freeVarCount = freeVarIdx.size();
+        int[] freeVar = new int[freeVarCount];
+        for (int i = 0; i < freeVarCount; i++) {
+            freeVar[i] = freeVarIdx.get(i);
+        }
+
+        double[] wmc = W.mv(M.mv(c));
+        double[] r = new double[freeVarCount];
+        for (int i = 0; i < freeVarCount; i++) {
+            int fi = freeVar[i];
+            r[i] = g[fi] + (cauchy[fi] - x[fi]) * theta - wmc[fi];
+        }
+
+        Matrix WZ = W.row(freeVar);
+        double[] v = M.mv(WZ.tv(r));
+        Matrix N = WZ.ata().mul(-thetaInverse);
+        N = M.mm(N);
+        int n1 = N.nrows();
+        for (int i = 0; i < n1; i++) {
+            N.add(i, i, 1.0);
+        }
+
+        DecompositionSolver lu = N.lu();
+        v = lu.solve(new ArrayRealVector(v)).toArray();
+
+        double[] wzv = WZ.mv(v);
+
+        double[] du = new double[freeVarCount];
+        for (int i = 0; i < freeVarCount; i++) {
+            du[i] = -thetaInverse * (r[i] + wzv[i] * thetaInverse);
+        }
+        double alphaStar = findAlpha(cauchy, du, l, u, freeVar);
+        double[] dStar = new double[freeVarCount];
+        for (int i = 0; i < freeVarCount; i++) {
+            dStar[i] = du[i] * alphaStar;
+        }
+
+        double[] subspaceMin = cauchy.clone();
+        for (int i = 0; i < freeVarCount; i++) {
+            subspaceMin[freeVar[i]] += dStar[i];
+        }
+
+        return subspaceMin;
+    }
+
+    /**
+     * Find the alpha* parameter, a positive scalar. See Equation 5.8 on page 11.
+     * @param cauchy cauchy point
+     * @param du intermediate results used to find alpha*
+     * @param freeVar the indices of free variable
+     */
+    private static double findAlpha(double[] cauchy, double[] du, double[] l, double[] u, int[] freeVar) {
+        double alphaStar = 1.0;
+        int n = freeVar.length;
+
+        for (int i = 0; i < n; i++) {
+            int fi = freeVar[i];
+            alphaStar = du[i] > 0 ? min(alphaStar, (u[fi] - cauchy[fi]) / du[i]) : min(alphaStar, (l[fi] - cauchy[fi]) / du[i]);
+        }
+
+        return alphaStar;
+    }
+
+    /**
+     * Returns the infinity norm of gradient.
+     * @param x the parameter vector
+     * @param g the gradient vector
+     */
+    private static double gnorm(double[] x, double[] g, double[] l, double[] u) {
+        double norm = 0.0;
+        int n = x.length;
+
+        for (int i = 0; i < n; i++) {
+            double gi = g[i];
+            if (gi < 0)
+                gi = max(x[i] - u[i], gi);
+            else
+                gi = min(x[i] - l[i], gi);
+            norm = max(norm, abs(gi));
+        }
+
+        return norm;
+    }
+
+    /**
+     * Clamps the values of v to stay within the pre-defined bounds.
+     * @param v the vector to be adjusted
+     */
+    private static void clampToBound(double[] v, double[] l, double[] u) {
+        int n = v.length;
+
+        for (int i = 0; i < n; i++) {
+            if (v[i] > u[i])
+                v[i] = u[i];
+            else if (v[i] < l[i])
+                v[i] = l[i];
+        }
+    }
+
+    public interface MultivariateFunction {
+        /**
+         * Computes the value of the function at x.
+         * @param x a real vector.
+         * @return the function value.
+         */
+        double f(double[] x);
+    }
+
+    public interface DifferentiableMultivariateFunction extends MultivariateFunction {
+        /** A number close to zero, between machine epsilon and its square root. */
+        double EPSILON = Double.parseDouble(System.getProperty("smile.gradient.epsilon", "1E-8"));
+
+        /**
+         * Computes the value and gradient at x. The default implementation uses finite differences to calculate the gradient.
+         * When possible, the subclass should compute the gradient analytically.
+         * @param x a real vector.
+         * @param gradient the output variable of gradient.
+         * @return the function value.
+         */
+        default double g(double[] x, double[] gradient) {
+            double fx = f(x);
+
+            int n = x.length;
+            double[] xh = new double[n];
+            for (int i = 0; i < n; i++) {
+                System.arraycopy(x, 0, xh, 0, n);
+                double xi = x[i];
+                double h = EPSILON * Math.abs(xi);
+                if (h == 0.0) {
+                    h = EPSILON;
+                }
+                xh[i] = xi + h; // trick to reduce finite-precision error.
+                h = xh[i] - xi;
+
+                double fh = f(xh);
+                xh[i] = xi;
+                gradient[i] = (fh - fx) / h;
+            }
+
+            return fx;
+        }
+    }
+
+    private static int[] sortWithIndex(double[] array) {
+        List<Pair<Integer, Double>> sortedPairs = IntStream.range(0, array.length).mapToObj(idx -> Pair.create(idx, array[idx]))
+                .sorted(Comparator.comparingDouble(Pair::getValue)).collect(Collectors.toList());
+        double[] sortedDoubles = sortedPairs.stream().mapToDouble(Pair::getValue).toArray();
+        int[] sortedIndices = sortedPairs.stream().mapToInt(Pair::getKey).toArray();
+        System.arraycopy(sortedDoubles, 0, array, 0, array.length);
+        return sortedIndices;
+    }
+
+    /**
+     * This is a substitute for SMILE's matrix implementation.
+     */
+    private static class Matrix {
+
+        private RealMatrix d;
+        private int m;
+        private int n;
+
+        /**
+         * Constructor of zero matrix.
+         * @param m the number of rows.
+         * @param n the number of columns.
+         */
+        public Matrix(RealMatrix delegate) {
+            d = delegate;
+            this.m = delegate.getRowDimension();
+            this.n = delegate.getColumnDimension();
+        }
+
+        /**
+         * Constructor of zero matrix.
+         * @param m the number of rows.
+         * @param n the number of columns.
+         */
+        public Matrix(int m, int n) {
+            d = new Array2DRowRealMatrix(m, n);
+            this.m = m;
+            this.n = n;
+        }
+
+        /**
+         * Sets {@code A[i,j] = x}.
+         * @param i the row index.
+         * @param j the column index.
+         * @param x the matrix cell value.
+         */
+        public void set(int i, int j, double x) {
+            d.setEntry(i, j, x);
+        }
+
+        /**
+         * Returns matrix multiplication {@code A' * B}.
+         * @param B the operand.
+         * @return the multiplication.
+         */
+        public Matrix tm(Matrix B) {
+            if (d.getRowDimension() != B.d.getRowDimension()) {
+                throw new IllegalArgumentException(String.format("Matrix multiplication A' * B: %d x %d vs %d x %d", m, n, B.m, B.n));
+            }
+
+            return new Matrix(d.transpose().multiply(B.d));
+        }
+
+        /**
+         * Returns {@code A' * A}.
+         * @return {@code A' * A}.
+         */
+        public Matrix ata() {
+            Matrix C = new Matrix(d.transpose().multiply(d));
+            // C.mm(TRANSPOSE, 1this, NO_TRANSPOSE, this);
+            // C.uplo(LOWER);
+            return C;
+        }
+
+        /**
+         * Returns {@code A[i,j]}.
+         * @param i the row index.
+         * @param j the column index.
+         * @return the matrix cell value.
+         */
+        public double get(int i, int j) {
+            return d.getEntry(i, j);
+        }
+
+        /**
+         * Returns the inverse of matrix. For pseudo inverse, use QRDecomposition.
+         * @return the inverse of matrix.
+         */
+        public Matrix inverse() {
+            return new Matrix(MatrixUtils.inverse(d));
+        }
+
+        /**
+         * Returns Matrix-vector multiplication {@code A' * x}.
+         * @param x the vector.
+         * @return the matrix-vector multiplication {@code A' * x}.
+         */
+        public double[] tv(double[] x) {
+            return d.transpose().operate(x);
+        }
+
+        /**
+         * Returns the quadratic form {@code x' * A * x}. The left upper sub-matrix of A is used in the computation based on the
+         * size of x.
+         * @param x the vector.
+         * @return the quadratic form.
+         */
+        public double xAx(double[] x) {
+            if (m != n) {
+                throw new IllegalArgumentException(String.format("The matrix is not square: %d x %d", m, n));
+            }
+
+            if (n != x.length) {
+                throw new IllegalArgumentException(String.format("Matrix: %d x %d, Vector: %d", m, n, x.length));
+            }
+
+            double[] Ax = mv(x);
+            return new ArrayRealVector(x).dotProduct(new ArrayRealVector(Ax));
+        }
+
+        /**
+         * Returns the matrix-vector multiplication {@code A * x}.
+         * @param x the vector.
+         * @return the matrix-vector multiplication {@code A * x}.
+         */
+        public double[] mv(double[] x) {
+            return d.operate(x);
+        }
+
+        /**
+         * Returns the i-th row. Negative index -i means the i-th row from the end.
+         * @param i the row index.
+         * @return the row.
+         */
+        public double[] row(int i) {
+            i = i < 0 ? m + i : i;
+            return d.getRow(i);
+        }
+
+        /**
+         * Returns the matrix of selected rows. Negative index -i means the i-th row from the end.
+         * @param rows the row indices.
+         * @return the sub-matrix.
+         */
+        public Matrix row(int... rows) {
+            double[][] data = new double[rows.length][n];
+
+            for (int i = 0; i < rows.length; i++) {
+                double[] row = row(rows[i]);
+                System.arraycopy(row, 0, data[i], 0, n);
+            }
+
+            return new Matrix(new Array2DRowRealMatrix(data, false));
+        }
+
+        public Matrix mul(double e) {
+            return new Matrix(d.scalarMultiply(e));
+        }
+
+        public Matrix mm(Matrix n2) {
+            return new Matrix(d.multiply(n2.d));
+        }
+
+        public int nrows() {
+            return m;
+        }
+
+        public void add(int m, int n, double e) {
+            d.addToEntry(m, n, e);
+        }
+
+        public DecompositionSolver lu() {
+            return new LUDecomposition(d).getSolver();
+        }
+
+    }
+}

jplag/src/main/java/de/jplag/clustering/algorithm/BayesianOptimization.java

@@ -0,0 +1,199 @@
+package de.jplag.clustering.algorithm;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Spliterator;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+import org.apache.commons.math3.distribution.NormalDistribution;
+import org.apache.commons.math3.exception.DimensionMismatchException;
+import org.apache.commons.math3.exception.OutOfRangeException;
+import org.apache.commons.math3.linear.ArrayRealVector;
+import org.apache.commons.math3.linear.RealVector;
+import org.apache.commons.math3.random.HaltonSequenceGenerator;
+import org.apache.commons.math3.random.RandomVectorGenerator;
+
+/**
+ * Maximizes a function using bayesian optimization.
+ */
+public class BayesianOptimization {
+
+    private static final int STOP_AFTER_CONSECUTIVE_RANDOM_PICKS = 3;
+    private static final int MAX_NON_ZERO_ACQ_FN_EVALS_PER_ITERATION = 50;
+    private static final int MAXIMUM_ACQ_FN_EVALS_PER_ITERATION = 1000;
+
+    private RealVector minima;
+    private RealVector maxima;
+    private int maxEvaluations;
+    private int initialPoints;
+    private double noise;
+    private RealVector lengthScale;
+    private boolean debug = false;
+
+    /**
+     * @param minima of the explored parameters
+     * @param maxima of the explored parameters
+     * @param initPoints points that are initially sampled for exploration
+     * @param maxEvaluations maximal evaluations of the fitted function
+     * @param noise of the explored function
+     * @param lengthScale width parameter for the matern kernel
+     */
+    public BayesianOptimization(RealVector minima, RealVector maxima, int initPoints, int maxEvaluations, double noise, RealVector lengthScale) {
+        if (minima.getDimension() != maxima.getDimension()) {
+            throw new DimensionMismatchException(minima.getDimension(), maxima.getDimension());
+        }
+        if (initPoints < 1 || initPoints > maxEvaluations) {
+            throw new OutOfRangeException(initPoints, 1, maxEvaluations);
+        }
+        this.maxima = maxima;
+        this.minima = minima;
+        this.initialPoints = initPoints;
+        this.maxEvaluations = maxEvaluations;
+        this.noise = noise;
+        this.lengthScale = lengthScale;
+    }
+
+    public void setDebugLogging(boolean debug) {
+        this.debug = debug;
+    }
+
+    private Stream<RealVector> sampleSolutionSpace() {
+        RandomVectorGenerator generator = new HaltonSequenceGenerator(minima.getDimension());
+        RealVector size = maxima.subtract(minima);
+        return Stream.generate(generator::nextVector).map(haltonPoint -> new ArrayRealVector(haltonPoint).ebeMultiply(size).add(minima));
+
+    }
+
+    private GaussianProcess fit(List<RealVector> listOfCoordinates, List<Double> observations) {
+        return GaussianProcess.fit(listOfCoordinates, observations.stream().mapToDouble(Double::doubleValue).toArray(), noise, true,
+                lengthScale.toArray());
+    }
+
+    private double acquisitionFunction(GaussianProcess gaussianProcess, double[] coordinates, double yMax) {
+        // expected improvement
+        double[] meanAndStandardDeviation = gaussianProcess.predict(new ArrayRealVector(coordinates));
+        double mean = meanAndStandardDeviation[0];
+        double standardDeviation = meanAndStandardDeviation[1];
+        double neededImprovement = mean - yMax;
+        double normalizedNeededImprovement = neededImprovement / standardDeviation;
+        NormalDistribution normalDistribution = new NormalDistribution(null, 0, 1);
+        return neededImprovement * normalDistribution.cumulativeProbability(normalizedNeededImprovement)
+                + standardDeviation * normalDistribution.density(normalizedNeededImprovement);
+    }
+
+    private static <T> Optional<T> getNext(Spliterator<T> spliterator) {
+        final List<T> result = new ArrayList<>(1);
+
+        if (spliterator.tryAdvance(result::add)) {
+            return Optional.of(result.get(0));
+        } else {
+            return Optional.empty();
+        }
+    }
+
+    /**
+     * Numerically optimize acquisition function
+     */
+    private RealVector maxAcq(GaussianProcess gaussianProcess, double yMax, Spliterator<RealVector> samples, double[] randomPicks) {
+        double bestScore = Double.NEGATIVE_INFINITY;
+        double[] bestSolution = getNext(samples).orElseThrow().toArray();
+        double[] min = minima.toArray();
+        double[] max = maxima.toArray();
+
+        int nonZeroAcquisitions = 0;
+        for (int i = 0; i < MAXIMUM_ACQ_FN_EVALS_PER_ITERATION && nonZeroAcquisitions < MAX_NON_ZERO_ACQ_FN_EVALS_PER_ITERATION; i++) {
+            double[] location = getNext(samples).orElseThrow().toArray();
+            if (acquisitionFunction(gaussianProcess, location, yMax) == 0)
+                continue;
+            nonZeroAcquisitions++;
+            double acquisition = -BFGS.minimize(coordinates -> -acquisitionFunction(gaussianProcess, coordinates, yMax), 5, location, min, max,
+                    0.00001, 1000);
+            // Sometimes result is out of bounds (might be due to numerical errors?)
+            for (int j = 0; j < location.length; j++) {
+                location[j] = Math.min(max[j], location[j]);
+                location[j] = Math.max(min[j], location[j]);
+            }
+            if (acquisition > bestScore) {
+                bestSolution = location;
+                bestScore = acquisition;
+            }
+        }
+        if (nonZeroAcquisitions == 0) {
+            randomPicks[0]++;
+        } else {
+            randomPicks[0] = 0;
+        }
+        return new ArrayRealVector(bestSolution);
+    }
+
+    /**
+     * Optimizes a real-valued function and returns a result associated with the optimal value.
+     * @param <T> type of the result
+     * @param objectiveFunction function to optimize
+     * @return result
+     */
+    public <T> OptimizationResult<T> maximize(Function<RealVector, OptimizationResult<T>> objectiveFunction) {
+        List<Double> observations = new ArrayList<>(maxEvaluations);
+        List<RealVector> testedCoordinates = new ArrayList<>(maxEvaluations);
+        OptimizationResult<T> best = null;
+
+        // the first couple of executions are reserved for exploration
+        sampleSolutionSpace().limit(initialPoints).forEach(testedCoordinates::add);
+
+        Spliterator<RealVector> poiSampler = sampleSolutionSpace().spliterator();
+        double[] zeroAcquisitionsCounter = new double[1];
+
+        while (observations.size() < maxEvaluations && zeroAcquisitionsCounter[0] < STOP_AFTER_CONSECUTIVE_RANDOM_PICKS) {
+            int idx = observations.size();
+            RealVector coordinates;
+            if (idx < testedCoordinates.size()) {
+                // hard coded exploration
+                coordinates = testedCoordinates.get(idx);
+            } else {
+                // GPR
+                GaussianProcess gpr = fit(testedCoordinates, observations);
+                if (debug) {
+                    System.out.println(gpr.toString(minima, maxima, 100, 25, 0));
+                }
+                coordinates = maxAcq(gpr, best.score, poiSampler, zeroAcquisitionsCounter);
+                testedCoordinates.add(coordinates);
+            }
+            OptimizationResult<T> result = objectiveFunction.apply(coordinates);
+            result.params = coordinates;
+            observations.add(result.getScore());
+            if (best == null) {
+                best = result;
+            } else if (result.score > best.score) {
+                best = result;
+            }
+        }
+
+        return best;
+    }
+
+    public static final class OptimizationResult<T> {
+
+        private double score;
+        private T value;
+        private RealVector params;
+
+        public OptimizationResult(double score, T value) {
+            this.score = score;
+            this.value = value;
+        }
+
+        public T getValue() {
+            return value;
+        }
+
+        public double getScore() {
+            return score;
+        }
+
+        public RealVector getParams() {
+            return params;
+        }
+    }
+}

jplag/src/main/java/de/jplag/clustering/algorithm/GaussianProcess.java

@@ -0,0 +1,197 @@
+package de.jplag.clustering.algorithm;
+
+import java.text.MessageFormat;
+import java.util.List;
+import java.util.OptionalInt;
+import java.util.function.DoubleToIntFunction;
+import java.util.stream.DoubleStream;
+import java.util.stream.IntStream;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.linear.ArrayRealVector;
+import org.apache.commons.math3.linear.CholeskyDecomposition;
+import org.apache.commons.math3.linear.DiagonalMatrix;
+import org.apache.commons.math3.linear.RealMatrix;
+import org.apache.commons.math3.linear.RealVector;
+import org.apache.commons.math3.stat.StatUtils;
+
+/**
+ * Implementation of a gaussian process with a matern kernel. This class can be used to fit any real-valued function
+ * with noisy evaluations. Predictions come in the form of expectations and standard deviations.
+ */
+public class GaussianProcess {
+
+    private List<RealVector> listOfCoordinates;
+    private RealVector weight;
+    private double mean;
+    private double standardDeviation;
+    private CholeskyDecomposition cholesky;
+    private RealVector lengthScale;
+
+    private GaussianProcess(List<RealVector> listOfCoordinates, RealVector weight, double mean, double standardDeviation,
+            CholeskyDecomposition cholesky, RealVector lengthScale) {
+        this.listOfCoordinates = listOfCoordinates;
+        this.weight = weight;
+        this.mean = mean;
+        this.standardDeviation = standardDeviation;
+        this.cholesky = cholesky;
+        this.lengthScale = lengthScale;
+    }
+
+    /**
+     * @param coordinates coordinate to predict at
+     * @return array containing the predicted [mean, standard deviation] at x.
+     */
+    public double[] predict(RealVector coordinates) {
+        RealVector kernelizedCoordinates = maternKernel(listOfCoordinates, coordinates, lengthScale);
+        RealVector kernelMatrixTimesCoordinates = cholesky.getSolver().solve(kernelizedCoordinates);
+
+        double predictedMean = weight.dotProduct(kernelizedCoordinates);
+        double predictedStandardDeviation = Math
+                .sqrt(maternKernel(coordinates, coordinates, lengthScale) - kernelMatrixTimesCoordinates.dotProduct(kernelizedCoordinates));
+
+        double[] out = new double[2];
+        out[0] = predictedMean * this.standardDeviation + this.mean;
+        out[1] = predictedStandardDeviation * this.standardDeviation;
+
+        if (Double.isNaN(out[1])) {
+            out[1] = 0;
+        }
+
+        return out;
+    }
+
+    private static final double SQRT_5 = Math.sqrt(5);
+
+    /**
+     * Fit Gaussian Process using a matern kernel.
+     * @param observedCoordinates
+     * @param observations expected to have zero mean, unit variance if normalize is false
+     * @param noise variance of noise in Y
+     * @param normalize if Y should be normalized
+     * @param lengthScale X values are divided by these values before calculating their euclidean distance. If all entries
+     * are equal the kernel is isometric.
+     */
+    public static GaussianProcess fit(List<RealVector> observedCoordinates, double[] observations, double noise, boolean normalize,
+            double[] lengthScale) {
+        if (observedCoordinates.size() < 1)
+            throw new IllegalArgumentException("Observed coordinates are empty");
+        if (observedCoordinates.size() != observations.length) {
+            throw new IllegalArgumentException(MessageFormat.format("Observed coordinates and observations are of different dimensions {0} and {1}",
+                    observedCoordinates.size(), observations.length));
+        }
+        OptionalInt brokenXIndex = IntStream.range(1, observedCoordinates.size())
+                .filter(i -> observedCoordinates.get(i).getDimension() != observedCoordinates.get(i - 1).getDimension()).findFirst();
+        brokenXIndex.ifPresent(brokenIndex -> {
+            throw new IllegalArgumentException(MessageFormat.format(
+                    "Observed coordinates has different dimensions at index {0} ({2}) and index {1} ({3})", brokenIndex - 1, brokenXIndex,
+                    observedCoordinates.get(brokenIndex - 1).getDimension(), observedCoordinates.get(brokenIndex).getDimension()));
+        });
+        if (noise <= 0) {
+            throw new IllegalArgumentException(MessageFormat.format("noise must be strictly positive, got {0}", noise));
+        }
+        if (lengthScale.length != observedCoordinates.get(0).getDimension()) {
+            throw new IllegalArgumentException(MessageFormat.format("lengthScale is of different dimension {0} than the coordinates values {1}",
+                    lengthScale.length, observedCoordinates.get(0).getDimension()));
+        }
+        double mean = 0;
+        double standardDeviation = 1;
+        if (normalize) {
+            mean = StatUtils.mean(observations);
+            standardDeviation = Math.sqrt(StatUtils.variance(observations, mean));
+            for (int i = 0; i < observations.length; i++) {
+                observations[i] = (observations[i] - mean) / standardDeviation;
+            }
+        }
+        RealVector observationVector = new ArrayRealVector(observations);
+        RealVector lengthScaleVector = new ArrayRealVector(lengthScale);
+
+        RealMatrix k = maternKernel(observedCoordinates, lengthScaleVector);
+        k = k.add(new DiagonalMatrix(DoubleStream.generate(() -> noise).limit(observedCoordinates.size()).toArray(), false));
+
+        CholeskyDecomposition decomposition = new CholeskyDecomposition(k);
+        RealVector w = decomposition.getSolver().solve(observationVector);
+
+        return new GaussianProcess(observedCoordinates, w, mean, standardDeviation, decomposition, lengthScaleVector);
+    }
+
+    /**
+     * Matern kernel for nu=2.5 (we get a twice differentiable gp)
+     */
+    private static RealMatrix maternKernel(List<RealVector> observedCoordinates, RealVector lengthScale) {
+        RealMatrix k = new Array2DRowRealMatrix(observedCoordinates.size(), observedCoordinates.size());
+        for (int i = 0; i < observedCoordinates.size(); i++) {
+            RealVector left = observedCoordinates.get(i);
+            for (int j = i; j < observedCoordinates.size(); j++) {
+                RealVector right = observedCoordinates.get(j);
+                double maternVal = maternKernel(left, right, lengthScale);
+                k.setEntry(i, j, maternVal);
+                k.setEntry(j, i, maternVal);
+            }
+        }
+        return k;
+    }
+
+    private static RealVector maternKernel(List<RealVector> observedCoordinates, RealVector vector, RealVector lengthScale) {
+        RealVector out = new ArrayRealVector(observedCoordinates.size());
+        for (int i = 0; i < observedCoordinates.size(); i++) {
+            out.setEntry(i, maternKernel(observedCoordinates.get(i), vector, lengthScale));
+        }
+        return out;
+    }
+
+    private static double maternKernel(RealVector left, RealVector right, RealVector lengthScale) {
+        double dist = left.ebeDivide(lengthScale).getDistance(right.ebeDivide(lengthScale));
+        dist *= SQRT_5;
+        return (1 + dist) * Math.exp(-dist);
+    }
+
+    /**
+     * ascii graph for debugging
+     */
+    public String toString(RealVector min, RealVector max, int width, int height, double minY) {
+        char[][] out = new char[height][width];
+        for (int i = 0; i < height; i++) {
+            for (int j = 0; j < width; j++) {
+                out[i][j] = ' ';
+            }
+        }
+        double[] mean = new double[width];
+        double[] std = new double[width];
+        for (int i = 0; i < width; i++) {
+            double t = (i + 0.5) / width;
+            RealVector coordinates = min.add(max.subtract(min).mapMultiplyToSelf(t));
+            double[] meanStd = predict(coordinates);
+            mean[i] = meanStd[0];
+            std[i] = meanStd[1];
+        }
+        double[] upperBorder = IntStream.range(0, width).mapToDouble(i -> mean[i] + std[i]).toArray();
+        double[] lowerBorder = IntStream.range(0, width).mapToDouble(i -> mean[i] - std[i]).toArray();
+        double minVal = Math.max(DoubleStream.of(lowerBorder).min().orElseThrow(), minY);
+        double maxVal = DoubleStream.of(upperBorder).max().orElseThrow();
+        double range = maxVal - minVal;
+        DoubleToIntFunction toCharPos = x -> (int) Math.round(height - height * (x - minVal) / range);
+        int[] upperCharPos = DoubleStream.of(upperBorder).mapToInt(toCharPos).toArray();
+        int[] lowerCharPos = DoubleStream.of(lowerBorder).mapToInt(toCharPos).toArray();
+        int[] mindCharPos = DoubleStream.of(mean).mapToInt(toCharPos).toArray();
+
+        for (int i = 0; i < width; i++) {
+            if (upperCharPos[i] > 0 && upperCharPos[i] < height)
+                out[upperCharPos[i]][i] = '-';
+            if (lowerCharPos[i] > 0 && lowerCharPos[i] < height)
+                out[lowerCharPos[i]][i] = '-';
+            if (mindCharPos[i] > 0 && mindCharPos[i] < height)
+                out[mindCharPos[i]][i] = '+';
+        }
+
+        StringBuilder stringBuilder = new StringBuilder();
+        for (int i = 0; i < height; i++) {
+            stringBuilder.append(out[i]);
+            if (i < 98) {
+                stringBuilder.append('\n');
+            }
+        }
+        return stringBuilder.toString();
+    }
+
+}

jplag/src/main/java/de/jplag/clustering/algorithm/GenericClusteringAlgorithm.java

@@ -0,0 +1,13 @@
+package de.jplag.clustering.algorithm;
+
+import java.util.Collection;
+
+import org.apache.commons.math3.linear.RealMatrix;
+
+/**
+ * Interface for algorithms that can perform clustering using only a symmetric matrix of similarities. The integers in
+ * the returned collection of integer collections denote the rows / columns of the similarity matrix.
+ */
+public interface GenericClusteringAlgorithm {
+    Collection<Collection<Integer>> cluster(RealMatrix similarityMatrix);
+}

jplag/src/main/java/de/jplag/clustering/algorithm/InterClusterSimilarity.java

@@ -0,0 +1,44 @@
+package de.jplag.clustering.algorithm;
+
+import java.util.List;
+import java.util.function.DoubleBinaryOperator;
+
+import org.apache.commons.math3.linear.RealMatrix;
+
+public enum InterClusterSimilarity {
+    MIN(Float.MAX_VALUE, Math::min),
+    MAX(Float.MIN_VALUE, Math::max),
+    AVERAGE(0, (a, b) -> a + b);
+
+    private final float neutralElement;
+    private final DoubleBinaryOperator accumulator;
+
+    private InterClusterSimilarity(float neutralElement, DoubleBinaryOperator accumulator) {
+        this.neutralElement = neutralElement;
+        this.accumulator = accumulator;
+    }
+
+    /**
+     * Calculates the distance between two clusters.
+     * @param leftCluster list of cluster indices in left cluster
+     * @param rightCluster list of cluster indices in right cluster
+     * @param similarityMatrix matrix containing similarities
+     * @return similarity between the two clusters
+     */
+    public float clusterSimilarity(List<Integer> leftCluster, List<Integer> rightCluster, RealMatrix similarityMatrix) {
+        float similarity = this.neutralElement;
+
+        for (int leftIndex = 0; leftIndex < leftCluster.size(); leftIndex++) {
+            int leftSubmission = leftCluster.get(leftIndex);
+            for (int rightIndex = 0; rightIndex < rightCluster.size(); rightIndex++) {
+                float submissionSimilarity = (float) similarityMatrix.getEntry(leftSubmission, rightCluster.get(rightIndex));
+                similarity = (float) this.accumulator.applyAsDouble(similarity, submissionSimilarity);
+            }
+        }
+
+        if (this == InterClusterSimilarity.AVERAGE) {
+            similarity /= leftCluster.size() * rightCluster.size();
+        }
+        return similarity;
+    }
+}

jplag/src/main/java/de/jplag/clustering/algorithm/SpectralClustering.java

@@ -0,0 +1,141 @@
+package de.jplag.clustering.algorithm;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.DoubleStream;
+import java.util.stream.IntStream;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.linear.ArrayRealVector;
+import org.apache.commons.math3.linear.DefaultRealMatrixChangingVisitor;
+import org.apache.commons.math3.linear.DiagonalMatrix;
+import org.apache.commons.math3.linear.EigenDecomposition;
+import org.apache.commons.math3.linear.RealMatrix;
+import org.apache.commons.math3.linear.RealVector;
+import org.apache.commons.math3.ml.clustering.Cluster;
+import org.apache.commons.math3.ml.clustering.Clusterable;
+import org.apache.commons.math3.ml.clustering.Clusterer;
+import org.apache.commons.math3.ml.clustering.KMeansPlusPlusClusterer;
+
+import de.jplag.clustering.ClusteringOptions;
+import de.jplag.clustering.ClusteringResult;
+
+/**
+ * Spectral clustering is a clustering algorithm for graph data. Each node is represented as k-dimensional vector,
+ * afterwards k-Means is used to generate a clustering with k on that representation. This implementation uses Bayesian
+ * Optimization to find an appropriate number for k.
+ */
+public class SpectralClustering implements GenericClusteringAlgorithm {
+
+    private static final double MULTIPLICITY_EPSILON = 0.05;
+    private final ClusteringOptions options;
+
+    public SpectralClustering(ClusteringOptions options) {
+        this.options = options;
+    }
+
+    @Override
+    public Collection<Collection<Integer>> cluster(RealMatrix similarityMatrix) {
+        // Calculate points to cluster according to "On spectral clustering: analysis and an algorithm" by Ng, Jordan & Weiss
+        // 2001
+        int dimension = similarityMatrix.getRowDimension();
+
+        // We don't use the similarity function, we already have some kind of similarity
+        RealMatrix weights = similarityMatrix.copy();
+        weights.walkInOptimizedOrder(new DefaultRealMatrixChangingVisitor() {
+            @Override
+            public double visit(int row, int column, double value) {
+                if (row == column)
+                    return 0;
+                return similarityMatrix.getEntry(row, column);
+            }
+        });
+
+        DiagonalMatrix diagonalPowMinus1Over2 = new DiagonalMatrix(dimension);
+        diagonalPowMinus1Over2.walkInOptimizedOrder(new DefaultRealMatrixChangingVisitor() {
+            @Override
+            public double visit(int row, int column, double value) {
+                if (row != column)
+                    return 0;
+                return 1 / Math.sqrt(weights.getRowVector(row).getL1Norm());
+            }
+        });
+
+        RealMatrix identity = new Array2DRowRealMatrix(dimension, dimension);
+        identity.walkInOptimizedOrder(new DefaultRealMatrixChangingVisitor() {
+            @Override
+            public double visit(int row, int column, double value) {
+                return row == column ? 1 : 0;
+            }
+        });
+        RealMatrix laplacian = identity.subtract(diagonalPowMinus1Over2.multiply(weights).multiply(diagonalPowMinus1Over2));
+        EigenDecomposition eigenDecomposition = new EigenDecomposition(laplacian);
+
+        List<Integer> eigenValueIds = new ArrayList<>(dimension);
+        for (int i = 0; i < dimension; i++) {
+            eigenValueIds.add(i);
+        }
+        eigenValueIds.sort(Comparator.comparingDouble(eigenDecomposition::getRealEigenvalue));
+
+        // find number of clusters as the multiplicity of eigenvalue 0
+        int minClusters = Math.max(2, (int) DoubleStream.of(eigenDecomposition.getRealEigenvalues()).filter(x -> x < MULTIPLICITY_EPSILON).count());
+        int maxClusters = (int) Math.ceil(dimension / 2.0);
+
+        // Find number of clusters using bayesian optimization
+        RealVector lengthScale = new ArrayRealVector(1, options.getSpectralKernelBandwidth());
+        BayesianOptimization bo = new BayesianOptimization(new ArrayRealVector(1, minClusters), new ArrayRealVector(1, maxClusters),
+                options.getSpectralMinRuns(), options.getSpectralMaxRuns(), options.getSpectralGaussianProcessVariance(), lengthScale);
+        // bo.debug = true;
+        BayesianOptimization.OptimizationResult<Collection<Collection<Integer>>> bayesianOptimizationResult = bo.maximize(r -> {
+            int clusters = (int) Math.round(r.getEntry(0));
+            clusters = Math.max(minClusters, clusters);
+            clusters = Math.min(maxClusters, clusters);
+            Collection<Collection<Integer>> clustering = cluster(clusters, dimension, eigenValueIds, eigenDecomposition);
+            ClusteringResult<Integer> modularityRes = ClusteringResult.fromIntegerCollections(new ArrayList<>(clustering), similarityMatrix);
+            return new BayesianOptimization.OptimizationResult<>(modularityRes.getWorth((a, b) -> (float) similarityMatrix.getEntry(a, b)),
+                    clustering);
+        });
+
+        return bayesianOptimizationResult.getValue();
+    }
+
+    private Collection<Collection<Integer>> cluster(int numberOfClusters, int dimension, List<Integer> eigenValueIds, EigenDecomposition ed) {
+        RealMatrix concatenatedEigenVectors = new Array2DRowRealMatrix(dimension, numberOfClusters);
+        concatenatedEigenVectors.walkInOptimizedOrder(new DefaultRealMatrixChangingVisitor() {
+            public double visit(int row, int column, double value) {
+                int eigenVectorId = eigenValueIds.get(column);
+                RealVector eigenVector = ed.getEigenvector(eigenVectorId);
+                return eigenVector.getEntry(row);
+            };
+        });
+
+        List<ClusterableEigenVector> normRows = IntStream.range(0, dimension).filter(i -> concatenatedEigenVectors.getRowVector(i).getNorm() > 0)
+                .mapToObj(row -> new ClusterableEigenVector(row, concatenatedEigenVectors.getRowVector(row).unitVector()))
+                .collect(Collectors.toList());
+
+        Clusterer<ClusterableEigenVector> clusterer = new KMeansPlusPlusClusterer<>(numberOfClusters, options.getSpectralMaxKMeansIterationPerRun());
+        List<? extends Cluster<ClusterableEigenVector>> clusters = clusterer.cluster(normRows);
+        return clusters.stream().map(cluster -> {
+            return cluster.getPoints().stream().map(eigenVector -> eigenVector.id).collect(Collectors.toList());
+        }).collect(Collectors.toList());
+    }
+
+    private static class ClusterableEigenVector implements Clusterable {
+        private int id;
+        private double[] eigenVector;
+
+        public ClusterableEigenVector(int id, RealVector eigenVector) {
+            this.id = id;
+            this.eigenVector = eigenVector.toArray();
+        }
+
+        @Override
+        public double[] getPoint() {
+            return eigenVector;
+        }
+    }
+
+}

jplag/src/main/java/de/jplag/clustering/preprocessors/CumulativeDistributionFunctionPreprocessor.java

@@ -0,0 +1,53 @@
+package de.jplag.clustering.preprocessors;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.linear.DefaultRealMatrixChangingVisitor;
+import org.apache.commons.math3.linear.DefaultRealMatrixPreservingVisitor;
+import org.apache.commons.math3.linear.RealMatrix;
+import org.apache.commons.math3.random.EmpiricalDistribution;
+
+import de.jplag.clustering.ClusteringPreprocessor;
+import de.jplag.clustering.PreprocessorHelper;
+
+/**
+ * Multiplies entries of the similarity matrix by their cumulative probability. - Weights that are very low will be
+ * (close to) zero - Weights that are very high will be (close to) unchanged
+ */
+public class CumulativeDistributionFunctionPreprocessor implements ClusteringPreprocessor {
+
+    private PreprocessorHelper helper = new PreprocessorHelper();
+
+    @Override
+    public double[][] preprocessSimilarities(double[][] similarityMatrix) {
+        RealMatrix similarity = new Array2DRowRealMatrix(similarityMatrix, true);
+        int connections = (similarity.getColumnDimension() * (similarity.getColumnDimension() - 1)) / 2;
+        EmpiricalDistribution dist = new EmpiricalDistribution(Math.max(100, connections / 100));
+        double[] allWeights = new double[connections];
+        similarity.walkInOptimizedOrder(new DefaultRealMatrixPreservingVisitor() {
+
+            int index = 0;
+
+            @Override
+            public void visit(int row, int column, double value) {
+                // collect upper triangle
+                if (row > column) {
+                    allWeights[this.index++] = value;
+                }
+            }
+        });
+        dist.load(allWeights);
+        similarity.walkInOptimizedOrder(new DefaultRealMatrixChangingVisitor() {
+            @Override
+            public double visit(int row, int column, double value) {
+                return dist.cumulativeProbability(value) * value;
+            }
+        });
+        return helper.removeDisconnectedEntries(similarity.getData());
+    }
+
+    @Override
+    public int originalIndexOf(int result) {
+        return helper.postProcessResult(result);
+    }
+
+}

jplag/src/main/java/de/jplag/clustering/preprocessors/PercentileThresholdProcessor.java

@@ -0,0 +1,51 @@
+package de.jplag.clustering.preprocessors;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.linear.DefaultRealMatrixPreservingVisitor;
+import org.apache.commons.math3.stat.descriptive.rank.Percentile;
+import org.apache.commons.math3.stat.descriptive.rank.Percentile.EstimationType;
+
+import de.jplag.clustering.ClusteringPreprocessor;
+
+/**
+ * Suppresses all similarities below a given percentile.
+ */
+public class PercentileThresholdProcessor implements ClusteringPreprocessor {
+
+    private float percentile;
+    private ThresholdPreprocessor thresholdPreprocessor;
+
+    public PercentileThresholdProcessor(float percentile) {
+        this.percentile = percentile;
+    }
+
+    @Override
+    public double[][] preprocessSimilarities(double[][] similarityMatrix) {
+        Array2DRowRealMatrix similarity = new Array2DRowRealMatrix(similarityMatrix, false);
+        Percentile percentileEstimator = new Percentile().withEstimationType(EstimationType.R_2);
+        int connections = (similarity.getColumnDimension() * (similarity.getColumnDimension() - 1)) / 2;
+        double[] allWeights = new double[connections];
+        similarity.walkInOptimizedOrder(new DefaultRealMatrixPreservingVisitor() {
+
+            int index = 0;
+
+            @Override
+            public void visit(int row, int column, double value) {
+                // collect upper triangle
+                if (row > column) {
+                    allWeights[this.index++] = value;
+                }
+            }
+        });
+        double threshold = percentileEstimator.evaluate(allWeights, percentile);
+        thresholdPreprocessor = new ThresholdPreprocessor(threshold);
+
+        return thresholdPreprocessor.preprocessSimilarities(similarityMatrix);
+    }
+
+    @Override
+    public int originalIndexOf(int result) {
+        return thresholdPreprocessor.originalIndexOf(result);
+    }
+
+}

jplag/src/main/java/de/jplag/clustering/preprocessors/ThresholdPreprocessor.java

@@ -0,0 +1,43 @@
+package de.jplag.clustering.preprocessors;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.linear.DefaultRealMatrixChangingVisitor;
+import org.apache.commons.math3.linear.RealMatrix;
+
+import de.jplag.clustering.ClusteringPreprocessor;
+import de.jplag.clustering.PreprocessorHelper;
+
+/**
+ * Suppresses all similarities below a given threshold.
+ */
+public class ThresholdPreprocessor implements ClusteringPreprocessor {
+
+    private double threshold;
+    private PreprocessorHelper helper = new PreprocessorHelper();
+
+    public ThresholdPreprocessor(double threshold) {
+        this.threshold = threshold;
+    }
+
+    @Override
+    public double[][] preprocessSimilarities(double[][] similarityMatrix) {
+        RealMatrix similarity = new Array2DRowRealMatrix(similarityMatrix, true);
+        similarity.walkInOptimizedOrder(new DefaultRealMatrixChangingVisitor() {
+            @Override
+            public double visit(int row, int column, double value) {
+                return value >= threshold ? value : 0;
+            }
+        });
+        return helper.removeDisconnectedEntries(similarity.getData());
+    }
+
+    @Override
+    public int originalIndexOf(int result) {
+        return helper.postProcessResult(result);
+    }
+
+    public double getThreshold() {
+        return threshold;
+    }
+
+}

jplag/src/main/java/de/jplag/options/JPlagOptions.java

@@ -11,6 +11,7 @@ import java.util.Optional;
 import java.util.Set;
 
 import de.jplag.Language;
+import de.jplag.clustering.ClusteringOptions;
 import de.jplag.strategy.ComparisonMode;
 
 public class JPlagOptions {
@@ -64,7 +65,7 @@ public class JPlagOptions {
 
     /**
      * Tunes the comparison sensitivity by adjusting the minimum token required to be counted as matching section. A smaller
-     * <n> increases the sensitivity but might lead to more false-positves.
+     * <n> increases the sensitivity but might lead to more false-positives.
      */
     private Integer minimumTokenMatch;
 
@@ -115,6 +116,11 @@ public class JPlagOptions {
      */
     private Verbosity verbosity;
 
+    /**
+     * Clustering options
+     */
+    private ClusteringOptions clusteringOptions = new ClusteringOptions.Builder().build();
+
     /**
      * Constructor with required attributes.
      */
@@ -187,6 +193,10 @@ public class JPlagOptions {
         return debugParser;
     }
 
+    public ClusteringOptions getClusteringOptions() {
+        return this.clusteringOptions;
+    }
+
     public void setBaseCodeSubmissionName(String baseCodeSubmissionName) {
         if (baseCodeSubmissionName == null || baseCodeSubmissionName.isEmpty()) {
             this.baseCodeSubmissionName = null;
@@ -279,6 +289,10 @@ public class JPlagOptions {
         this.verbosity = verbosity;
     }
 
+    public void setClusteringOptions(ClusteringOptions clusteringOptions) {
+        this.clusteringOptions = clusteringOptions;
+    }
+
     private boolean hasFileSuffixes() {
         return fileSuffixes != null && fileSuffixes.length > 0;
     }
@@ -286,4 +300,5 @@ public class JPlagOptions {
     private boolean hasMinimumTokenMatch() {
         return minimumTokenMatch != null;
     }
+
 }

jplag/src/main/java/de/jplag/options/SimilarityMetric.java

@@ -4,18 +4,24 @@ import java.util.function.Function;
 
 import de.jplag.JPlagComparison;
 
-public enum SimilarityMetric {
-    AVG(JPlagComparison::similarity),
-    MIN(JPlagComparison::minimalSimilarity),
-    MAX(JPlagComparison::maximalSimilarity);
+public enum SimilarityMetric implements Function<JPlagComparison, Float> {
+    AVG(it -> it.similarity()),
+    MIN(it -> it.minimalSimilarity()),
+    MAX(it -> it.maximalSimilarity()),
+    INTERSECTION(it -> (float) it.getNumberOfMatchedTokens());
 
     private final Function<JPlagComparison, Float> similarityFunction;
 
-    SimilarityMetric(Function<JPlagComparison, Float> determinePercentage) {
+    private SimilarityMetric(Function<JPlagComparison, Float> determinePercentage) {
         this.similarityFunction = determinePercentage;
     }
 
     public boolean isAboveThreshold(JPlagComparison comparison, float similarityThreshold) {
         return similarityFunction.apply(comparison) >= similarityThreshold;
     }
+
+    @Override
+    public Float apply(JPlagComparison comparison) {
+        return similarityFunction.apply(comparison);
+    }
 }

jplag/src/main/java/de/jplag/reporting/Report.java

@@ -159,7 +159,7 @@ public class Report { // Mostly legacy code with some minor improvements.
             htmlFile.print("<TR><TD BGCOLOR=\"" + col + "\"><FONT COLOR=\"" + col + "\">-</FONT>");
             htmlFile.print("<TD><A HREF=\"javascript:ZweiFrames('match" + index + "-0.html#" + i + "',2,'match" + index + "-1.html#" + i
                     + "',3)\" NAME=\"" + i + "\">");
-            htmlFile.print(new String(startA.file.getBytes()));
+            htmlFile.print(new String(startA.getFile().getBytes()));
 
             if (options.getLanguage().usesIndex()) {
                 htmlFile.print("(" + startA.getIndex() + "-" + endA.getIndex() + ")");
@@ -169,7 +169,7 @@ public class Report { // Mostly legacy code with some minor improvements.
 
             htmlFile.print("<TD><A HREF=\"javascript:ZweiFrames('match" + index + "-0.html#" + i + "',2,'match" + index + "-1.html#" + i
                     + "',3)\" NAME=\"" + i + "\">");
-            htmlFile.print(startB.file);
+            htmlFile.print(startB.getFile());
 
             if (options.getLanguage().usesIndex()) {
                 htmlFile.print("(" + startB.getIndex() + "-" + endB.getIndex());
@@ -289,7 +289,7 @@ public class Report { // Mostly legacy code with some minor improvements.
             Token start = tokens.getToken(onematch.getStart(j == 0));
             Token end = tokens.getToken((onematch.getStart(j == 0)) + onematch.getLength() - 1);
             for (int fileIndex = 0; fileIndex < files.length; fileIndex++) {
-                if (start.file.equals(files[fileIndex]) && text[fileIndex] != null) {
+                if (start.getFile().equals(files[fileIndex]) && text[fileIndex] != null) {
                     String tmp = "<FONT color=\"" + Color.getHexadecimalValue(x) + "\">" + (j == 1 ? "<div style=\"position:absolute;left:0\">" : "")
                             + "<A HREF=\"javascript:ZweiFrames('match" + i + "-" + (1 - j) + ".html#" + x + "'," + (3 - j) + ",'match" + i
                             + "-top.html#" + x + "',1)\"><IMG SRC=\"" + PICS[j] + "\" ALT=\"other\" " + "BORDER=\"0\" ALIGN=\""
@@ -314,7 +314,7 @@ public class Report { // Mostly legacy code with some minor improvements.
                 Token end = tokens.getToken(match.getStartOfFirst() + match.getLength() - 1);
 
                 for (int fileIndex = 0; fileIndex < files.length; fileIndex++) {
-                    if (start.file.equals(files[fileIndex]) && text[fileIndex] != null) {
+                    if (start.getFile().equals(files[fileIndex]) && text[fileIndex] != null) {
                         String tmp = "<font color=\"#C0C0C0\"><EM>";
                         // beginning of the colorblock
                         markupList.put(new MarkupText(fileIndex, start.getLine() - 1, start.getColumn() - 1, tmp, false), null);
@@ -698,7 +698,7 @@ public class Report { // Mostly legacy code with some minor improvements.
             Token end = tokens.getToken((currentMatch.getStart(j == 0)) + currentMatch.getLength() - 1);
 
             for (int y = 0; y < files.length; y++) {
-                if (start.file.equals(files[y]) && text[y] != null) {
+                if (start.getFile().equals(files[y]) && text[y] != null) {
                     hilf = "<FONT color=\"" + Color.getHexadecimalValue(x) + "\">" + (j == 1 ? "<div style=\"position:absolute;left:0\">" : "")
                             + "<A HREF=\"javascript:ZweiFrames('match" + i + "-" + (1 - j) + ".html#" + x + "'," + (3 - j) + ",'match" + i
                             + "-top.html#" + x + "',1)\"><IMG SRC=\"" + PICS[j] + "\" ALT=\"other\" " + "BORDER=\"0\" ALIGN=\""
@@ -731,7 +731,7 @@ public class Report { // Mostly legacy code with some minor improvements.
                 Token end = tokens.getToken(currentMatch.getStartOfFirst() + currentMatch.getLength() - 1);
 
                 for (int y = 0; y < files.length; y++) {
-                    if (start.file.equals(files[y]) && text[y] != null) {
+                    if (start.getFile().equals(files[y]) && text[y] != null) {
                         hilf = ("<font color=\"#C0C0C0\"><EM>");
                         // position the icon and the beginning of the colorblock
                         if (text[y][start.getLine() - 1].endsWith("<font color=\"#000000\">")) {

jplag/src/main/java/de/jplag/reporting2/JsonReport.java

@@ -0,0 +1,36 @@
+package de.jplag.reporting2;
+
+import java.io.File;
+import java.util.List;
+
+import de.jplag.JPlagResult;
+import de.jplag.reporting2.jsonfactory.JsonFactory;
+import de.jplag.reporting2.reportobject.ReportObjectFactory;
+import de.jplag.reporting2.reportobject.model.JPlagReport;
+
+// ReportImplementation -> JsonReport
+
+/**
+ * A report generator which reports the JPlagResult in Json format.
+ */
+public class JsonReport implements Report {
+
+    @Override
+    public List<String> getReportStrings(JPlagResult result) {
+        JPlagReport report = ReportObjectFactory.getReportObject(result);
+        return JsonFactory.getJsonStrings(report);
+    }
+
+    @Override
+    public boolean saveReport(JPlagResult result, String path) {
+        JPlagReport report = ReportObjectFactory.getReportObject(result);
+        File dir = new File(path);
+        if (!dir.exists()) {
+            if (!dir.mkdir()) {
+                System.out.println("Failed to create dir.");
+            }
+        }
+        return JsonFactory.saveJsonFiles(report, path);
+    }
+
+}

jplag/src/main/java/de/jplag/reporting2/Report.java

@@ -0,0 +1,29 @@
+package de.jplag.reporting2;
+
+import java.util.List;
+
+import de.jplag.JPlagResult;
+
+// ReportStrategy -> Report
+
+/**
+ * Strategy interface for reporting. A report generator should implement this interface.
+ */
+public interface Report {
+
+    /**
+     * This function returns a list containing the report objects as simple strings. The first element is the string of the
+     * overview object. Each following element is a string of a comparison report object.
+     * @param result The result of a JPlag comparison
+     * @return A list containing report objects to string. First element is Overview. Other elements are comparisons.
+     */
+    List<String> getReportStrings(JPlagResult result);
+
+    /**
+     * Creates and saves JPlag report files to the disk.
+     * @param result The result of a JPlag comparison.
+     * @param path Path to the directory where the report should be saved.
+     * @return True if the process is successful, otherwise false.
+     */
+    boolean saveReport(JPlagResult result, String path);
+}

jplag/src/main/java/de/jplag/reporting2/jsonfactory/JsonFactory.java

@@ -0,0 +1,57 @@
+package de.jplag.reporting2.jsonfactory;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import de.jplag.reporting2.reportobject.model.ComparisonReport;
+import de.jplag.reporting2.reportobject.model.JPlagReport;
+
+/**
+ * Factory class, responsible for creating Json strings and writing them to files.
+ */
+public class JsonFactory {
+
+    private static final ObjectMapper mapper = new ObjectMapper();
+
+    /**
+     * Uses Jackson to create Json Strings from JPlagReport object.
+     * @return A list, first element is Json String of Overview object. The rest elements are Json Strings of Comparison
+     * objects.
+     */
+    public static List<String> getJsonStrings(JPlagReport jPlagReport) {
+        List<String> jsonReports = new ArrayList<>();
+        try {
+            jsonReports.add(mapper.writeValueAsString(jPlagReport.getOverviewReport()));
+            for (ComparisonReport comparisonReport : jPlagReport.getComparisons()) {
+                jsonReports.add(mapper.writeValueAsString(comparisonReport));
+            }
+        } catch (JsonProcessingException e) {
+            System.out.println("Error converting object to json " + e.getMessage());
+        }
+        return jsonReports;
+    }
+
+    /**
+     * Creates Json Files for the given JPlagReport and saves them in the given folder.
+     * @return A boolean, representing whether the process was successful.
+     */
+    public static boolean saveJsonFiles(JPlagReport jPlagReport, String folderPath) {
+        try {
+            ObjectMapper mapper = new ObjectMapper();
+            mapper.writeValue(Path.of(folderPath, "overview.json").toFile(), jPlagReport.getOverviewReport());
+            for (ComparisonReport report : jPlagReport.getComparisons()) {
+                String name = report.getFirstSubmissionId().concat("-").concat(report.getSecondSubmissionId()).concat(".json");
+                mapper.writeValue(Path.of(folderPath, name).toFile(), report);
+            }
+        } catch (IOException e) {
+            System.out.println("Failed to save json files: " + e.getMessage());
+            return false;
+        }
+        return true;
+    }
+}

jplag/src/main/java/de/jplag/reporting2/reportobject/ReportObjectFactory.java

@@ -0,0 +1,189 @@
+package de.jplag.reporting2.reportobject;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import de.jplag.*;
+import de.jplag.reporting2.reportobject.model.*;
+import de.jplag.reporting2.reportobject.model.Match;
+
+/**
+ * Factory class, responsible for converting a JPlagResult object to Overview and Comparison DTO classes.
+ */
+public class ReportObjectFactory {
+
+    /**
+     * Converts a JPlagResult to a JPlagReport.
+     * @return JPlagReport for the given JPlagResult.
+     */
+    public static JPlagReport getReportObject(JPlagResult result) {
+        OverviewReport overviewReport = generateOverviewReport(result);
+        List<ComparisonReport> comparisons = generateComparisonReports(result);
+        return new JPlagReport(overviewReport, comparisons);
+    }
+
+    /**
+     * Generates an Overview DTO of a JPlagResult.
+     */
+    private static OverviewReport generateOverviewReport(JPlagResult result) {
+        List<JPlagComparison> comparisons = result.getComparisons();
+        OverviewReport overviewReport = new OverviewReport();
+
+        overviewReport.setSubmissionFolderPath(result.getOptions().getRootDirectoryNames());
+
+        String baseCodePath = result.getOptions().hasBaseCode() ? result.getOptions().getBaseCodeSubmissionName().orElse("") : "";
+        overviewReport.setBaseCodeFolderPath(baseCodePath);
+
+        overviewReport.setLanguage(result.getOptions().getLanguage().getName());
+        overviewReport.setFileExtensions(List.of(result.getOptions().getFileSuffixes()));
+        overviewReport.setSubmissionIds(extractSubmissionNames(comparisons));
+        overviewReport.setFailedSubmissionNames(List.of());  // No number of failed submissions
+        overviewReport.setExcludedFiles(result.getOptions().getExcludedFiles());
+        overviewReport.setMatchSensitivity(result.getOptions().getMinimumTokenMatch());
+        overviewReport.setDateOfExecution(getDate());
+        overviewReport.setExecutionTime(result.getDuration());
+        overviewReport.setComparisonNames(getComparisonNames(comparisons));
+        overviewReport.setMetrics(getMetrics(result));
+        overviewReport.setClusters(getClusters(result));
+
+        return overviewReport;
+    }
+
+    /**
+     * Generates detailed ComparisonReport DTO for each comparison in a JPlagResult.
+     * @return A list with ComparisonReport DTOs.
+     */
+    private static List<ComparisonReport> generateComparisonReports(JPlagResult result) {
+        List<ComparisonReport> comparisons = new ArrayList<>();
+        result.getComparisons().forEach(comparison -> comparisons.add( //
+                new ComparisonReport(comparison.getFirstSubmission().getName(), //
+                        comparison.getSecondSubmission().getName(), //
+                        comparison.similarity(), //
+                        getFilesForSubmission(comparison.getFirstSubmission()), //
+                        getFilesForSubmission(comparison.getSecondSubmission()), //
+                        convertMatchesToReportMatches(result, comparison, comparison.getMatches()) //
+                )));
+        return comparisons;
+    }
+
+    private static List<Match> convertMatchesToReportMatches(JPlagResult result, JPlagComparison comparison, List<de.jplag.Match> matches) {
+        return matches.stream().map(match -> convertMatchToReportMatch(comparison, match, result.getOptions().getLanguage().usesIndex()))
+                .collect(Collectors.toList());
+    }
+
+    /**
+     * Gets the names of all submissions.
+     * @return A list containing all submission names.
+     */
+    private static List<String> extractSubmissionNames(List<JPlagComparison> comparisons) {
+        HashSet<String> names = new HashSet<>();
+        comparisons.forEach(comparison -> {
+            names.add(comparison.getFirstSubmission().getName());
+            names.add(comparison.getSecondSubmission().getName());
+        });
+        return new ArrayList<>(names);
+    }
+
+    /**
+     * Gets the names of all comparison.
+     * @return A list containing all comparisons.
+     */
+    private static List<String> getComparisonNames(List<JPlagComparison> comparisons) {
+        List<String> names = new ArrayList<>();
+        comparisons.forEach(
+                comparison -> names.add(String.join("-", comparison.getFirstSubmission().getName(), comparison.getSecondSubmission().getName())));
+        return names;
+    }
+
+    // Currently, only one metric can be obtained.
+
+    /**
+     * Gets the used metric in a JPlag comparison.
+     * @return A list contains Metric DTOs.
+     */
+    private static List<Metric> getMetrics(JPlagResult result) {
+        List<Metric> metrics = new ArrayList<>();
+        metrics.add(new Metric(result.getOptions().getSimilarityMetric().name(), result.getOptions().getSimilarityThreshold(),
+                intArrayToList(result.getSimilarityDistribution()), getTopComparisons(result.getComparisons())));
+        return metrics;
+    }
+
+    /**
+     * Converts JPlagComparison to a DTO for displaying only comparisons. See
+     * {@link #generateComparisonReports(JPlagResult)} for a more detailed representation of a comparison.
+     * @return List containing TopComparison DTOs.
+     */
+    private static List<TopComparison> getTopComparisons(List<JPlagComparison> comparisons) {
+        List<TopComparison> topComparisons = new ArrayList<>();
+        comparisons.forEach(comparison -> topComparisons.add(
+                new TopComparison(comparison.getFirstSubmission().getName(), comparison.getSecondSubmission().getName(), comparison.similarity())));
+        return topComparisons;
+    }
+
+    /**
+     * Converts files of a submission to FilesOFSubmission DTO.
+     * @return A list containing FilesOfSubmission DTOs.
+     */
+    private static List<FilesOfSubmission> getFilesForSubmission(Submission submission) {
+        return submission.getFiles().stream().map(file -> new FilesOfSubmission(file.getName(), readFileLines(file))).collect(Collectors.toList());
+    }
+
+    /**
+     * Converts a JPlag Match object to a Match DTO.
+     * @param comparison The comparison from which the match originates.
+     * @param match The match to be converted.
+     * @param usesIndex Indicates whether the language uses indexes.
+     * @return A Match DTO.
+     */
+    private static Match convertMatchToReportMatch(JPlagComparison comparison, de.jplag.Match match, Boolean usesIndex) {
+        TokenList tokensFirst = comparison.getFirstSubmission().getTokenList();
+        TokenList tokensSecond = comparison.getSecondSubmission().getTokenList();
+        Token startTokenFirst = tokensFirst.getToken(match.getStartOfFirst());
+        Token endTokenFirst = tokensFirst.getToken(match.getStartOfFirst() + match.getLength() - 1);
+        Token startTokenSecond = tokensSecond.getToken(match.getStartOfSecond());
+        Token endTokenSecond = tokensSecond.getToken(match.getStartOfSecond() + match.getLength() - 1);
+
+        int startFirst = usesIndex ? startTokenFirst.getIndex() : startTokenFirst.getLine();
+        int endFirst = usesIndex ? endTokenFirst.getIndex() : endTokenFirst.getLine();
+        int startSecond = usesIndex ? startTokenSecond.getIndex() : startTokenSecond.getLine();
+        int endSecond = usesIndex ? endTokenSecond.getIndex() : endTokenSecond.getLine();
+        int tokens = match.getLength();
+
+        return new Match(startTokenFirst.getFile(), startTokenSecond.getFile(), startFirst, endFirst, startSecond, endSecond, tokens);
+    }
+
+    // TODO implement after PR Read clustering #281
+    private static List<Cluster> getClusters(JPlagResult result) {
+        // List<ClusteringResult<Submission>> clusters = result.getClusteringResult();
+        // return clusters.map( c -> new Cluster(getAvgSimilarity, getStrength, c.getMembers().map(Submission::getName)))
+        return List.of();
+    }
+
+    private static List<String> readFileLines(File file) {
+        List<String> lines = new ArrayList<>();
+        try (BufferedReader bufferedReader = new BufferedReader(new FileReader(file))) {
+            String line;
+            while ((line = bufferedReader.readLine()) != null) {
+                lines.add(line);
+            }
+        } catch (IOException exception) {
+            System.out.println("Could not read file: " + exception.getMessage());
+        }
+        return lines;
+    }
+
+    private static String getDate() {
+        SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yy");
+        Date date = new Date();
+        return dateFormat.format(date);
+    }
+
+    private static List<Integer> intArrayToList(int[] array) {
+        return Arrays.stream(array).boxed().collect(Collectors.toList());
+    }
+}

jplag/src/main/java/de/jplag/reporting2/reportobject/model/Cluster.java

@@ -0,0 +1,35 @@
+package de.jplag.reporting2.reportobject.model;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class Cluster {
+
+    @JsonProperty("average_similarity")
+    private final float averageSimilarity;
+
+    @JsonProperty("strength")
+    private final float strength;
+
+    @JsonProperty("members")
+    private final List<String> members;
+
+    public Cluster(float averageSimilarity, float strength, List<String> members) {
+        this.averageSimilarity = averageSimilarity;
+        this.strength = strength;
+        this.members = List.copyOf(members);
+    }
+
+    public float getAverageSimilarity() {
+        return averageSimilarity;
+    }
+
+    public float getStrength() {
+        return strength;
+    }
+
+    public List<String> getMembers() {
+        return members;
+    }
+}

jplag/src/main/java/de/jplag/reporting2/reportobject/model/ComparisonReport.java

@@ -0,0 +1,61 @@
+package de.jplag.reporting2.reportobject.model;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class ComparisonReport {
+
+    @JsonProperty("first_submission_id")
+    private final String firstSubmissionId;
+
+    @JsonProperty("second_submission_id")
+    private final String secondSubmissionId;
+
+    @JsonProperty("match_percentage")
+    private final float matchPercentage;
+
+    @JsonProperty("files_of_first_submission")
+    private final List<FilesOfSubmission> filesOfFirstSubmission;
+
+    @JsonProperty("files_of_second_submission")
+    private final List<FilesOfSubmission> filesOfSecondSubmission;
+
+    @JsonProperty("matches")
+    private final List<Match> matches;
+
+    public ComparisonReport(String firstSubmissionId, String secondSubmissionId, float matchPercentage,
+            List<FilesOfSubmission> filesOfFirstSubmission, List<FilesOfSubmission> filesOfSecondSubmission, List<Match> matches) {
+
+        this.firstSubmissionId = firstSubmissionId;
+        this.secondSubmissionId = secondSubmissionId;
+        this.matchPercentage = matchPercentage;
+        this.filesOfFirstSubmission = List.copyOf(filesOfFirstSubmission);
+        this.filesOfSecondSubmission = List.copyOf(filesOfSecondSubmission);
+        this.matches = matches;
+    }
+
+    public String getFirstSubmissionId() {
+        return firstSubmissionId;
+    }
+
+    public String getSecondSubmissionId() {
+        return secondSubmissionId;
+    }
+
+    public float getMatchPercentage() {
+        return matchPercentage;
+    }
+
+    public List<FilesOfSubmission> getFilesOfFirstSubmission() {
+        return filesOfFirstSubmission;
+    }
+
+    public List<FilesOfSubmission> getFilesOfSecondSubmission() {
+        return filesOfSecondSubmission;
+    }
+
+    public List<Match> getMatches() {
+        return matches;
+    }
+}

jplag/src/main/java/de/jplag/reporting2/reportobject/model/FilesOfSubmission.java

@@ -0,0 +1,28 @@
+package de.jplag.reporting2.reportobject.model;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class FilesOfSubmission {
+
+    @JsonProperty("file_name")
+    private final String fileName;
+
+    @JsonProperty("lines")
+    private final List<String> lines;
+
+    public FilesOfSubmission(String fileName, List<String> file_code) {
+        this.fileName = fileName;
+        this.lines = List.copyOf(file_code);
+    }
+
+    public String getFileName() {
+        return fileName;
+    }
+
+    public List<String> getLines() {
+        return lines;
+    }
+
+}

jplag/src/main/java/de/jplag/reporting2/reportobject/model/JPlagReport.java

@@ -0,0 +1,21 @@
+package de.jplag.reporting2.reportobject.model;
+
+import java.util.List;
+
+public class JPlagReport {
+    private final OverviewReport overviewReport;
+    private final List<ComparisonReport> comparisons;
+
+    public JPlagReport(OverviewReport overviewReport, List<ComparisonReport> comparisons) {
+        this.overviewReport = overviewReport;
+        this.comparisons = List.copyOf(comparisons);
+    }
+
+    public OverviewReport getOverviewReport() {
+        return overviewReport;
+    }
+
+    public List<ComparisonReport> getComparisons() {
+        return comparisons;
+    }
+}

jplag/src/main/java/de/jplag/reporting2/reportobject/model/Match.java

@@ -0,0 +1,65 @@
+package de.jplag.reporting2.reportobject.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class Match {
+
+    @JsonProperty("first_file_name")
+    private final String firstFileName;
+
+    @JsonProperty("second_file_name")
+    private final String secondFileName;
+
+    @JsonProperty("start_in_first")
+    private final int startInFirst;
+
+    @JsonProperty("end_in_first")
+    private final int endInFirst;
+
+    @JsonProperty("start_in_second")
+    private final int startInSecond;
+
+    @JsonProperty("end_in_second")
+    private final int endInSecond;
+
+    @JsonProperty("tokens")
+    private final int tokens;
+
+    public Match(String firstFileName, String secondFileName, int startInFirst, int endInFirst, int startInSecond, int endInSecond, int tokens) {
+        this.firstFileName = firstFileName;
+        this.secondFileName = secondFileName;
+        this.startInFirst = startInFirst;
+        this.endInFirst = endInFirst;
+        this.startInSecond = startInSecond;
+        this.endInSecond = endInSecond;
+        this.tokens = tokens;
+    }
+
+    public String getFirstFileName() {
+        return firstFileName;
+    }
+
+    public String getSecondFileName() {
+        return secondFileName;
+    }
+
+    public int getStartInFirst() {
+        return startInFirst;
+    }
+
+    public int getEndInFirst() {
+        return endInFirst;
+    }
+
+    public int getStartInSecond() {
+        return startInSecond;
+    }
+
+    public int getEndInSecond() {
+        return endInSecond;
+    }
+
+    public int getTokens() {
+        return tokens;
+    }
+}

jplag/src/main/java/de/jplag/reporting2/reportobject/model/Metric.java

@@ -0,0 +1,43 @@
+package de.jplag.reporting2.reportobject.model;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class Metric {
+
+    @JsonProperty("name")
+    private final String name;
+
+    @JsonProperty("threshold")
+    private final float threshold;
+
+    @JsonProperty("distribution")
+    private final List<Integer> distribution;
+
+    @JsonProperty("topComparisons")
+    private final List<TopComparison> topComparisons;
+
+    public Metric(String name, float threshold, List<Integer> distribution, List<TopComparison> topComparisons) {
+        this.name = name;
+        this.threshold = threshold;
+        this.distribution = List.copyOf(distribution);
+        this.topComparisons = List.copyOf(topComparisons);
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public float getThreshold() {
+        return threshold;
+    }
+
+    public List<Integer> getDistribution() {
+        return distribution;
+    }
+
+    public List<TopComparison> getTopComparisons() {
+        return topComparisons;
+    }
+}

jplag/src/main/java/de/jplag/reporting2/reportobject/model/OverviewReport.java

@@ -0,0 +1,174 @@
+package de.jplag.reporting2.reportobject.model;
+
+import java.util.List;
+import java.util.Set;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class OverviewReport {
+
+    @JsonProperty("submission_folder_path")
+    private List<String> submissionFolderPath;
+
+    @JsonProperty("base_code_folder_path")
+    private String baseCodeFolderPath;
+
+    @JsonProperty("language")
+    private String language;
+
+    @JsonProperty("file_extensions")
+    private List<String> fileExtensions;
+
+    @JsonProperty("submission_ids")
+    private List<String> submissionIds;
+
+    @JsonProperty("failed_submission_names")
+    private List<String> failedSubmissionNames;
+
+    @JsonProperty("excluded_files")
+    private Set<String> excludedFiles;
+
+    @JsonProperty("match_sensitivity")
+    private int matchSensitivity;
+
+    @JsonProperty("date_of_execution")
+    private String dateOfExecution;
+
+    @JsonProperty("execution_time")
+    private long executionTime;
+
+    @JsonProperty("comparison_names")
+    private List<String> comparisonNames;
+
+    @JsonProperty("metrics")
+    private List<Metric> metrics;
+
+    @JsonProperty("clusters")
+    private List<Cluster> clusters;
+
+    public OverviewReport() {
+        submissionFolderPath = List.of();
+        baseCodeFolderPath = "";
+        language = "";
+        fileExtensions = List.of();
+        submissionIds = List.of();
+        failedSubmissionNames = List.of();
+        excludedFiles = Set.of();
+        matchSensitivity = 0;
+        dateOfExecution = "";
+        executionTime = 0;
+        clusters = List.of();
+    }
+
+    /*******************************
+     * GETTERS
+     *******************************/
+
+    public List<String> getSubmissionFolderPath() {
+        return submissionFolderPath;
+    }
+
+    public String getBaseCodeFolderPath() {
+        return baseCodeFolderPath;
+    }
+
+    public String getLanguage() {
+        return language;
+    }
+
+    public List<String> getFileExtensions() {
+        return fileExtensions;
+    }
+
+    public List<String> getSubmissionIds() {
+        return submissionIds;
+    }
+
+    public List<String> getFailedSubmissionNames() {
+        return failedSubmissionNames;
+    }
+
+    public Set<String> getExcludedFiles() {
+        return excludedFiles;
+    }
+
+    public int getMatchSensitivity() {
+        return matchSensitivity;
+    }
+
+    public String getDateOfExecution() {
+        return dateOfExecution;
+    }
+
+    public long getExecutionTime() {
+        return executionTime;
+    }
+
+    public List<String> getComparisonNames() {
+        return comparisonNames;
+    }
+
+    public List<Metric> getMetrics() {
+        return metrics;
+    }
+
+    public List<Cluster> getClusters() {
+        return clusters;
+    }
+
+    /*******************************
+     * SETTERS
+     *******************************/
+
+    public void setSubmissionFolderPath(List<String> submissionFolderPath) {
+        this.submissionFolderPath = submissionFolderPath;
+    }
+
+    public void setBaseCodeFolderPath(String baseCodeFolderPath) {
+        this.baseCodeFolderPath = baseCodeFolderPath;
+    }
+
+    public void setLanguage(String language) {
+        this.language = language;
+    }
+
+    public void setFileExtensions(List<String> fileExtensions) {
+        this.fileExtensions = List.copyOf(fileExtensions);
+    }
+
+    public void setSubmissionIds(List<String> submissionIds) {
+        this.submissionIds = List.copyOf(submissionIds);
+    }
+
+    public void setFailedSubmissionNames(List<String> failedSubmissionNames) {
+        this.failedSubmissionNames = List.copyOf(failedSubmissionNames);
+    }
+
+    public void setExcludedFiles(Set<String> excludedFiles) {
+        this.excludedFiles = Set.copyOf(excludedFiles);
+    }
+
+    public void setMatchSensitivity(int matchSensitivity) {
+        this.matchSensitivity = matchSensitivity;
+    }
+
+    public void setDateOfExecution(String dateOfExecution) {
+        this.dateOfExecution = dateOfExecution;
+    }
+
+    public void setExecutionTime(long executionTime) {
+        this.executionTime = executionTime;
+    }
+
+    public void setComparisonNames(List<String> comparisonNames) {
+        this.comparisonNames = List.copyOf(comparisonNames);
+    }
+
+    public void setMetrics(List<Metric> metrics) {
+        this.metrics = List.copyOf(metrics);
+    }
+
+    public void setClusters(List<Cluster> clusters) {
+        this.clusters = List.copyOf(clusters);
+    }
+}

jplag/src/main/java/de/jplag/reporting2/reportobject/model/TopComparison.java

@@ -0,0 +1,34 @@
+package de.jplag.reporting2.reportobject.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class TopComparison {
+
+    @JsonProperty("first_submission")
+    private final String firstSubmission;
+
+    @JsonProperty("second_submission")
+    private final String secondSubmission;
+
+    @JsonProperty("match_percentage")
+    private final float matchPercentage;
+
+    public TopComparison(String firstSubmission, String secondSubmission, float matchPercentage) {
+        this.firstSubmission = firstSubmission;
+        this.secondSubmission = secondSubmission;
+        this.matchPercentage = matchPercentage;
+    }
+
+    public String getFirstSubmission() {
+        return firstSubmission;
+    }
+
+    public String getSecondSubmission() {
+        return secondSubmission;
+    }
+
+    public float getMatchPercentage() {
+        return matchPercentage;
+    }
+
+}

jplag/src/main/resources/de/jplag/messages.properties

@@ -11,3 +11,17 @@ CommandLineArgument.ShownComparisons=The maximum number of comparisons that will
 CommandLineArgument.SimilarityThreshold=Comparison similarity threshold [0-100]: All comparisons above this threshold will be saved
 CommandLineArgument.Subdirectory=Look in directories <root-dir>/*/<dir> for programs
 CommandLineArgument.Verbosity=Verbosity of the logging
+CommandLineArgument.ClusterEnable=Skips the clustering
+CommandLineArgument.ClusterAlgorithm=Which clustering algorithm to use. Agglomerative merges similar submissions bottom up. Spectral clustering is combined with Bayesian Optimization to execute the k-Means clustering algorithm multiple times, hopefully finding a "good" clustering automatically.
+CommandLineArgument.ClusterMetric=The metric used for clustering. AVG is intersection over union, MAX can expose some attempts of obfuscation.
+CommandLineArgument.ClusterSpectralBandwidth=Bandwidth of the matern kernel in the Gaussian Process used during the search for a good number of clusters for spectral clustering. If a good clustering result is found during the search, numbers of clusters that differ by something in range of the bandwidth are also expected to good.
+CommandLineArgument.ClusterSpectralNoise=The result of each run in the search for good clusterings are random. The noise level models the variance in the "worth" of these results. It also acts as a regularization constant.
+CommandLineArgument.ClusterSpectralMinRuns=Minimum number of k-Means executions during spectral clustering. With these initial clustering sizes are explored.
+CommandLineArgument.ClusterSpectralMaxRuns=Maximum number of k-Means executions during spectral clustering. Any execution after the initial runs tries to balance between exploration of unknown clustering sizes and exploitation of clustering sizes known as good.
+CommandLineArgument.ClusterSpectralKmeansIterations=Maximum number of iterations during each execution of the k-Means algorithm.
+CommandLineArgument.ClusterAgglomerativeThreshold=Only clusters with an inter-cluster-similarity greater than this threshold are merged during agglomerative clustering.
+CommandLineArgument.ClusterAgglomerativeInterClusterSimilarity=How to measure the similarity of two clusters during agglomerative clustering. Minimum, maximum or average similarity between the submissions in each cluster.
+CommandLineArgument.ClusterPreprocessingNone=Do not use any preprocessing before clustering. Not recommended for spectral clustering.
+CommandLineArgument.ClusterPreprocessingCdf=Before clustering, the value of the cumulative distribution function of all similarities is estimated. The similarities are multiplied with these estimates. This has the effect of supressing similarities that are low compared to other similarities.
+CommandLineArgument.ClusterPreprocessingPercentile=Any similarity smaller than the given percentile will be suppressed during clustering.
+CommandLineArgument.ClusterPreprocessingThreshold=Any similarity smaller than the given threshold value will be suppressed during clustering.

jplag/src/test/java/de/jplag/NewJavaFeaturesTest.java

@@ -18,13 +18,10 @@ public class NewJavaFeaturesTest extends TestBase {
         assertEquals(2, result.getNumberOfSubmissions());
         assertEquals(1, result.getComparisons().size());
         var comparison = result.getComparisons().get(0);
-        assertEquals(100, comparison.similarity(), Double.MIN_NORMAL);
+        assertEquals(92.1, comparison.similarity(), DELTA);
 
         // Five matches that each start at the same position:
         assertEquals(4, comparison.getMatches().size());
-        for (Match match : comparison.getMatches()) {
-            assertEquals(match.getStartOfFirst(), match.getStartOfSecond());
-        }
     }
 
 }

jplag/src/test/java/de/jplag/cli/ClusteringTest.java

@@ -0,0 +1,36 @@
+package de.jplag.cli;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+import de.jplag.CommandLineArgument;
+import de.jplag.clustering.Preprocessing;
+
+public class ClusteringTest extends CommandLineInterfaceTest {
+
+    private static final double EPSILON = 0.000001;
+
+    @Test
+    public void parsePercentilePreProcessor() {
+        String argument = CommandLineArgument.CLUSTER_PREPROCESSING_PERCENTILE.flag() + "=" + Float.toString(0.5f);
+        buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
+        assertEquals(Preprocessing.PERCENTILE, options.getClusteringOptions().getPreprocessor());
+        assertEquals(0.5, options.getClusteringOptions().getPreprocessorPercentile(), EPSILON);
+    }
+
+    @Test
+    public void parseCdfPreProcessor() {
+        String argument = CommandLineArgument.CLUSTER_PREPROCESSING_CDF.flag();
+        buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
+        assertEquals(Preprocessing.CUMULATIVE_DISTRIBUTION_FUNCTION, options.getClusteringOptions().getPreprocessor());
+    }
+
+    @Test
+    public void parseNoPreProcessor() {
+        String argument = CommandLineArgument.CLUSTER_PREPROCESSING_NONE.flag();
+        buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
+        assertEquals(Preprocessing.NONE, options.getClusteringOptions().getPreprocessor());
+    }
+
+}

jplag/src/test/java/de/jplag/clustering/ClusterTest.java

@@ -0,0 +1,53 @@
+package de.jplag.clustering;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+
+import org.junit.Test;
+
+public class ClusterTest {
+
+    private static final double EPSILON = 0.00001;
+    Cluster<Character> cluster;
+
+    @Test
+    public void testAverageSimilarity() {
+        cluster = new Cluster<>(List.of('a', 'b', 'c'), 0);
+        float averageSimilarity = cluster.averageSimilarity((a, b) -> {
+            return Math.abs((float) (((int) a) - ((int) b)));
+        });
+        assertEquals((1.f + 2.f + 1.f + 1.f + 2.f + 1.f) / 6, averageSimilarity, EPSILON);
+    }
+
+    @Test
+    public void testCommunityStrengthPerConnectionOneMember() {
+        cluster = new Cluster<>(List.of('a'), 10);
+        assertEquals(0.0, cluster.getCommunityStrengthPerConnection(), EPSILON);
+    }
+
+    @Test
+    public void testCommunityStrengthPerConnectionTwoMembers() {
+        cluster = new Cluster<>(List.of('a', 'b'), 10);
+        assertEquals(10.0, cluster.getCommunityStrengthPerConnection(), EPSILON);
+    }
+
+    @Test
+    public void testCommunityStrengthPerConnectionThreeMembers() {
+        cluster = new Cluster<>(List.of('a', 'b', 'c'), 10);
+        assertEquals(10.0 / 3, cluster.getCommunityStrengthPerConnection(), EPSILON);
+    }
+
+    @Test
+    public void testNormalizedCommunityStrength() {
+        cluster = new Cluster<>(List.of('a', 'b', 'c'), 10);
+        @SuppressWarnings("unchecked")
+        ClusteringResult<Character> clusteringResult = mock(ClusteringResult.class);
+        when(clusteringResult.getClusters()).thenReturn(List.of(cluster, cluster));
+        cluster.setClusteringResult(clusteringResult);
+        assertEquals(0.5, cluster.getNormalizedCommunityStrengthPerConnection(), EPSILON);
+    }
+
+}

jplag/src/test/java/de/jplag/clustering/ClusteringAdapterTest.java

@@ -0,0 +1,53 @@
+package de.jplag.clustering;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.commons.math3.linear.RealMatrix;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+
+import de.jplag.JPlagComparison;
+import de.jplag.Submission;
+import de.jplag.clustering.algorithm.GenericClusteringAlgorithm;
+
+public class ClusteringAdapterTest {
+
+    @Test
+    public void testClustering() {
+        List<Submission> submissions = IntStream.range(0, 4).mapToObj(x -> mock(Submission.class)).collect(Collectors.toList());
+        List<JPlagComparison> comparisons = new ArrayList<>(6);
+        for (int i = 0; i < submissions.size(); i++) {
+            for (int j = i + 1; j < submissions.size(); j++) {
+                JPlagComparison comparison = mock(JPlagComparison.class);
+                when(comparison.getFirstSubmission()).thenReturn(submissions.get(i));
+                when(comparison.getSecondSubmission()).thenReturn(submissions.get(j));
+                comparisons.add(comparison);
+            }
+        }
+
+        // Mock algorithm that returns everything in a single cluster
+        GenericClusteringAlgorithm algorithm = mock(GenericClusteringAlgorithm.class);
+        when(algorithm.cluster(any(RealMatrix.class))).then((InvocationOnMock invocation) -> {
+            RealMatrix arg = invocation.getArgument(0);
+            Collection<Collection<Integer>> result = List.of(IntStream.range(0, arg.getRowDimension()).boxed().collect(Collectors.toList()));
+            return result;
+        });
+
+        ClusteringAdapter clustering = new ClusteringAdapter(comparisons, x -> 0.f);
+        ClusteringResult<Submission> clusteringResult = clustering.doClustering(algorithm);
+
+        Collection<Collection<Submission>> expectedResult = List.of(submissions);
+
+        assertEquals(expectedResult, clusteringResult.getClusters().stream().map(Cluster::getMembers).collect(Collectors.toList()));
+    }
+
+}

jplag/src/test/java/de/jplag/clustering/ClusteringRealDataTest.java

@@ -0,0 +1,225 @@
+package de.jplag.clustering;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Scanner;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.linear.RealMatrix;
+import org.junit.Test;
+
+import de.jplag.clustering.algorithm.GenericClusteringAlgorithm;
+import de.jplag.clustering.algorithm.SpectralClustering;
+import de.jplag.clustering.preprocessors.CumulativeDistributionFunctionPreprocessor;
+
+/**
+ * These test are not meant to be run during normal unit testing. They can be used to test the clustering algorithms
+ * against data from the private pseudomized reports repository. These tests test PROBABILISTIC behavior, so use with
+ * caution!
+ */
+public class ClusteringRealDataTest {
+
+    private static class TestFile {
+
+        private String uri;
+        private Optional<List<String>> expected;
+
+        public TestFile(String uri, Optional<List<String>> expected) {
+            this.uri = uri;
+            this.expected = expected;
+        }
+    }
+
+    private static List<String> B_POSITIVE = Arrays.asList(new String[] {"Student (31)", "Student (223)"});
+    private static List<String> C_POSITIVE = Arrays.asList(new String[] {"Student (166)", "Student (212)", "Student (236)", "Student (229)"});
+
+    private static final TestFile[] OLD_CLUSTERING_DATA = {new TestFile("de/jplag/PseudonymizedReports/alt/A_1000_matches_max.csv", Optional.empty()),
+            new TestFile("de/jplag/PseudonymizedReports/alt/B_1000_matches_max.csv", Optional.of(B_POSITIVE)),
+            new TestFile("de/jplag/PseudonymizedReports/alt/C_1000_matches_max.csv", Optional.of(C_POSITIVE)),};
+
+    private static final TestFile[] NEW_CLUSTERING_DATA = {new TestFile("de/jplag/PseudonymizedReports/neu/A_matches_avg.csv", Optional.empty()),
+            new TestFile("de/jplag/PseudonymizedReports/neu/B_matches_avg.csv", Optional.of(B_POSITIVE)),
+            new TestFile("de/jplag/PseudonymizedReports/neu/C_matches_avg.csv", Optional.of(C_POSITIVE)),};
+
+    private String str(float f) {
+        return String.format("%.4f", f);
+    }
+
+    private URL loadFromClasspath(String file) throws FileNotFoundException {
+        URL url = getClass().getClassLoader().getResource(file);
+        if (url == null) {
+            assumeTrue(file + " not found. 'de/jpag/PseudonymizedReports' must contain the data from the PseudonymizedReports repository.", false);
+        }
+        return url;
+    }
+
+    private void doTesting(ReadResult readResult, Optional<List<String>> expected) {
+        RealMatrix clusteringSimilarity = new Array2DRowRealMatrix(readResult.similarity.getData());
+
+        /*
+         * AgglomerativeClustering.ClusteringOptions options = new AgglomerativeClustering.ClusteringOptions();
+         * options.minimalSimilarity = 0.15f; options.similarity = AgglomerativeClustering.InterClusterSimilarity.AVERAGE;
+         * ClusteringAlgorithm clusteringAlg = new AgglomerativeClustering(options);
+         */
+
+        SpectralClustering clusteringAlg = new SpectralClustering(ClusteringOptions.DEFAULTS);
+        ClusteringPreprocessor preprocessor = new CumulativeDistributionFunctionPreprocessor();
+        GenericClusteringAlgorithm preprocessedClusteringAlg = new PreprocessedClusteringAlgorithm(clusteringAlg, preprocessor);
+        Collection<Collection<Integer>> clustering = preprocessedClusteringAlg.cluster(clusteringSimilarity);
+        ClusteringResult<Integer> mRes = ClusteringResult.fromIntegerCollections(new ArrayList<>(clustering), readResult.similarity);
+        List<Cluster<Integer>> clusters = new ArrayList<>(mRes.getClusters());
+        clusters.sort(Comparator.comparingDouble(c -> -c.getNormalizedCommunityStrengthPerConnection()));
+
+        System.out.println("cs\tncsm\tavgSim\tcombined\tmembers");
+        for (Cluster<Integer> c : clusters) {
+            float ncsm = c.getCommunityStrengthPerConnection();
+            float avgSim = c.averageSimilarity((a, b) -> (float) readResult.similarity.getEntry(a, b));
+            System.out.println(str(c.getCommunityStrength()) + "\t" + str(ncsm) + "\t" + str(avgSim) + "\t"
+                    + c.getMembers().stream().map(readResult.mapping::unmap).collect(Collectors.toList()));
+        }
+        System.out.println("Community Strength: " + mRes.getCommunityStrength());
+        System.out.println("Clusters: " + clusters.size());
+
+        expected.ifPresent(expectedIdentifiers -> {
+            Set<String> expectedIdentifiersSet = new HashSet<>(expectedIdentifiers);
+            Set<String> bestClusters = clusters.get(0).getMembers().stream().map(readResult.mapping::unmap).collect(Collectors.toSet());
+            assertEquals(expectedIdentifiersSet, bestClusters);
+            System.out.println("hey");
+        });
+    }
+
+    @Test
+    public void aClusteringOld() throws FileNotFoundException, URISyntaxException {
+        for (TestFile testFile : OLD_CLUSTERING_DATA) {
+            URL url = loadFromClasspath(testFile.uri);
+            File file = new File(url.toURI());
+            ReadResult r = readOldCsv(file);
+            doTesting(r, testFile.expected);
+        }
+    }
+
+    @Test
+    public void aClusteringNew() throws FileNotFoundException, URISyntaxException {
+        for (TestFile testFile : NEW_CLUSTERING_DATA) {
+            URL url = loadFromClasspath(testFile.uri);
+            File file = new File(url.toURI());
+            ReadResult r = readNewCsv(file);
+            doTesting(r, testFile.expected);
+        }
+    }
+
+    private static class ReadComparison {
+        int left;
+        int right;
+        float similarity;
+    }
+
+    private static class ReadResult {
+        IntegerMapping<String> mapping;
+        RealMatrix similarity;
+    }
+
+    private static ReadResult readNewCsv(File fileName) throws FileNotFoundException {
+        IntegerMapping<String> mapping = new IntegerMapping<>(512);
+        List<ReadComparison> comparisons = new ArrayList<>(512);
+        try (CSVReader reader = new CSVReader(fileName, ";")) {
+            while (reader.hasNext()) {
+                List<String> records = reader.next();
+                if (records.isEmpty())
+                    continue;
+                String leftStudent = records.get(1);
+                String rightStudent = records.get(2);
+                String similarity = records.get(3);
+                ReadComparison comparison = new ReadComparison();
+                comparison.left = mapping.map(leftStudent);
+                comparison.right = mapping.map(rightStudent);
+                comparison.similarity = Float.parseFloat(similarity) / 100;
+                comparisons.add(comparison);
+            }
+        }
+        RealMatrix matrix = new Array2DRowRealMatrix(mapping.size(), mapping.size());
+        for (ReadComparison comparison : comparisons) {
+            matrix.setEntry(comparison.left, comparison.right, comparison.similarity);
+            matrix.setEntry(comparison.right, comparison.left, comparison.similarity);
+        }
+        ReadResult r = new ReadResult();
+        r.similarity = matrix;
+        r.mapping = mapping;
+        return r;
+    }
+
+    private static ReadResult readOldCsv(File fileName) throws FileNotFoundException {
+        IntegerMapping<String> mapping = new IntegerMapping<>(512);
+        List<ReadComparison> comparisons = new ArrayList<>(512);
+        try (CSVReader reader = new CSVReader(fileName, ";")) {
+            while (reader.hasNext()) {
+                List<String> records = reader.next();
+                if (records.isEmpty())
+                    continue;
+                Deque<String> stuff = new ArrayDeque<>(records);
+                String leftStudent = stuff.removeFirst();
+                int leftID = mapping.map(leftStudent);
+                while (stuff.size() >= 3) {
+                    ReadComparison comparison = new ReadComparison();
+                    comparison.left = leftID;
+                    stuff.removeFirst(); // comparison ID not needed
+                    String rightStudent = stuff.removeFirst();
+                    String similarity = stuff.removeFirst();
+                    comparison.right = mapping.map(rightStudent);
+                    comparison.similarity = Float.parseFloat(similarity) / 100;
+                    comparisons.add(comparison);
+                }
+            }
+        }
+        RealMatrix matrix = new Array2DRowRealMatrix(mapping.size(), mapping.size());
+        for (ReadComparison comparison : comparisons) {
+            matrix.setEntry(comparison.left, comparison.right, comparison.similarity);
+            matrix.setEntry(comparison.right, comparison.left, comparison.similarity);
+        }
+        ReadResult r = new ReadResult();
+        r.similarity = matrix;
+        r.mapping = mapping;
+        return r;
+    }
+
+    private static class CSVReader implements AutoCloseable {
+        private String delimiter;
+        private Scanner scanner;
+
+        private CSVReader(File fileName, String delimiter) throws FileNotFoundException {
+            this.delimiter = delimiter;
+            scanner = new Scanner(fileName);
+        }
+
+        @Override
+        public void close() {
+            scanner.close();
+        }
+
+        List<String> next() {
+            String line = scanner.nextLine();
+            String[] records = line.split(delimiter);
+            return Arrays.asList(records);
+        }
+
+        boolean hasNext() {
+            return scanner.hasNextLine();
+        }
+    }
+}

jplag/src/test/java/de/jplag/clustering/ClusteringResultTest.java

@@ -0,0 +1,56 @@
+package de.jplag.clustering;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.linear.RealMatrix;
+import org.junit.Test;
+
+public class ClusteringResultTest {
+
+    @Test
+    public void perfectClustering() {
+        RealMatrix similarity = new Array2DRowRealMatrix(4, 4);
+
+        // These are similar
+        setEntries(similarity, 0, 1, 1f);
+        setEntries(similarity, 2, 3, 1f);
+
+        // Others are dissimilar
+
+        ClusteringResult<Integer> result = ClusteringResult.fromIntegerCollections(List.of(List.of(0, 1), List.of(2, 3)), similarity);
+
+        // The maximum of the metric is 1 - 1/k for k clusters
+        assertEquals(0.5, result.getCommunityStrength(), 0.00001);
+    }
+
+    @Test
+    public void uniformClustering() {
+        RealMatrix similarity = new Array2DRowRealMatrix(4, 4);
+
+        // We'd obtain such weights by pre-selecting the clusters,
+        // then randomly picking two clusters and adding weight between random of each
+        // cluster
+
+        // These are similar
+        setEntries(similarity, 0, 1, 0.1f);
+        setEntries(similarity, 2, 3, 0.1f);
+
+        // Others are dissimilar
+        setEntries(similarity, 0, 2, 0.05f);
+        setEntries(similarity, 0, 3, 0.05f);
+        setEntries(similarity, 1, 2, 0.05f);
+        setEntries(similarity, 1, 3, 0.05f);
+
+        ClusteringResult<Integer> result = ClusteringResult.fromIntegerCollections(List.of(List.of(0, 1), List.of(2, 3)), similarity);
+
+        assertEquals(0.0, result.getCommunityStrength(), 0.00001);
+    }
+
+    private static void setEntries(RealMatrix matrix, int i, int j, double value) {
+        matrix.setEntry(i, j, value);
+        matrix.setEntry(j, i, value);
+    }
+}

jplag/src/test/java/de/jplag/clustering/algorithm/AgglomerativeClusteringTest.java

@@ -0,0 +1,17 @@
+package de.jplag.clustering.algorithm;
+
+import java.util.Collection;
+
+import org.junit.Test;
+
+public class AgglomerativeClusteringTest {
+
+    @Test
+    public void test() {
+        for (ClusteringData testData : ClusteringData.values()) {
+            AgglomerativeClustering clustering = new AgglomerativeClustering(testData.getOptions());
+            Collection<Collection<Integer>> result = clustering.cluster(testData.getSimilarity());
+            testData.assertValid(result);
+        }
+    }
+}

jplag/src/test/java/de/jplag/clustering/algorithm/BayesianOptimizationTest.java

@@ -0,0 +1,26 @@
+package de.jplag.clustering.algorithm;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.commons.math3.linear.ArrayRealVector;
+import org.apache.commons.math3.linear.RealVector;
+import org.junit.Test;
+
+public class BayesianOptimizationTest {
+
+    @Test
+    public void findParabolaMax() {
+        RealVector minima = new ArrayRealVector(new double[] {-20});
+        RealVector maxima = new ArrayRealVector(new double[] {10});
+        RealVector lengthScale = new ArrayRealVector(new double[] {5});
+        double maximumAt = 1;
+        BayesianOptimization bo = new BayesianOptimization(minima, maxima, 3, 15, 1.0 / 12.0, lengthScale);
+        double maximumPoint = bo.maximize(v -> {
+            double val = v.getEntry(0);
+            double result = -(val - maximumAt) * (val - maximumAt);
+            result += 0.2 * (Math.random() - 0.5);
+            return new BayesianOptimization.OptimizationResult<Double>(result, result);
+        }).getParams().getEntry(0);
+        assertEquals(maximumAt, maximumPoint, 1);
+    }
+}

jplag/src/test/java/de/jplag/clustering/algorithm/ClusteringData.java

@@ -0,0 +1,68 @@
+package de.jplag.clustering.algorithm;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.linear.RealMatrix;
+
+import de.jplag.clustering.ClusteringOptions;
+
+public enum ClusteringData {
+
+    FOUR_POINTS(() -> {
+        RealMatrix similarity = new Array2DRowRealMatrix(4, 4);
+        for (int i = 0; i < 4; i++) {
+            similarity.setEntry(i, i, 1);
+        }
+        // These are similar
+        setEntries(similarity, 0, 1, 0.5);
+        setEntries(similarity, 2, 3, 0.5f);
+
+        // Others are dissimilar
+        setEntries(similarity, 0, 2, 0.1f);
+        setEntries(similarity, 0, 3, 0.1f);
+        setEntries(similarity, 1, 2, 0.1f);
+        setEntries(similarity, 1, 3, 0.1f);
+
+        return similarity;
+    }, new int[][] {{0, 1}, {2, 3}}, new ClusteringOptions.Builder().agglomerativeThreshold(0.4f));
+
+    private final RealMatrix similarity;
+    private final Set<Set<Integer>> expected;
+    private final ClusteringOptions options;
+
+    private ClusteringData(Supplier<RealMatrix> similarity, int[][] expected, ClusteringOptions.Builder options) {
+        this.similarity = similarity.get();
+        this.expected = makeSets(
+                Arrays.stream(expected).map(intArray -> Arrays.stream(intArray).boxed().collect(Collectors.toList())).collect(Collectors.toList()));
+        this.options = options.build();
+    }
+
+    public ClusteringOptions getOptions() {
+        return options;
+    }
+
+    public RealMatrix getSimilarity() {
+        return similarity;
+    }
+
+    private Set<Set<Integer>> makeSets(Collection<? extends Collection<Integer>> collections) {
+        return collections.stream().map(HashSet::new).collect(Collectors.toSet());
+    }
+
+    public void assertValid(Collection<? extends Collection<Integer>> actual) {
+        assertEquals(this.name() + " not clustered correctly", expected, makeSets(actual));
+    }
+
+    private static void setEntries(RealMatrix matrix, int i, int j, double value) {
+        matrix.setEntry(i, j, value);
+        matrix.setEntry(j, i, value);
+    }
+}

jplag/src/test/java/de/jplag/clustering/algorithm/GaussianProcessTest.java

@@ -0,0 +1,37 @@
+package de.jplag.clustering.algorithm;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.math3.linear.ArrayRealVector;
+import org.apache.commons.math3.linear.RealVector;
+import org.junit.Test;
+
+public class GaussianProcessTest {
+
+    @Test
+    public void noisyLinearFunction() {
+        List<RealVector> X = new ArrayList<>();
+        double[] Y = new double[20];
+        int idx = 0;
+        for (int i = 0; i < 10; i++) {
+            RealVector x = new ArrayRealVector(1);
+            x.setEntry(0, i);
+            X.add(x);
+            X.add(x);
+            Y[idx++] = i + Math.random() - 0.5;
+            Y[idx++] = i + Math.random() - 0.5;
+        }
+        GaussianProcess gp = GaussianProcess.fit(X, Y, 1 / 12.0, true, new double[] {1});
+        for (int i = 0; i < 19; i++) {
+            RealVector vx = new ArrayRealVector(1);
+            double x = i / 2.0;
+            vx.setEntry(0, x);
+            double[] prediction = gp.predict(vx);
+            assertTrue("The prediction error can't very high", Math.abs(prediction[0] - x) < 1);
+            assertTrue("The standard deviation must be greater than 0", prediction[1] > 0);
+        }
+    }
+}

jplag/src/test/java/de/jplag/clustering/algorithm/SpectralClusteringTest.java

@@ -0,0 +1,17 @@
+package de.jplag.clustering.algorithm;
+
+import java.util.Collection;
+
+import org.junit.Test;
+
+public class SpectralClusteringTest {
+
+    @Test
+    public void test() {
+        for (ClusteringData testData : ClusteringData.values()) {
+            SpectralClustering clustering = new SpectralClustering(testData.getOptions());
+            Collection<Collection<Integer>> result = clustering.cluster(testData.getSimilarity());
+            testData.assertValid(result);
+        }
+    }
+}

jplag/src/test/java/de/jplag/clustering/preprocessors/CumulativeDistributionFunctionPreprocessorTest.java

@@ -0,0 +1,40 @@
+package de.jplag.clustering.preprocessors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class CumulativeDistributionFunctionPreprocessorTest extends PreprocessingTestBase {
+
+    private static final double EPSILON = 0.0000001;
+
+    CumulativeDistributionFunctionPreprocessor preprocessor;
+
+    @Before
+    public void init() {
+        preprocessor = new CumulativeDistributionFunctionPreprocessor();
+    }
+
+    @Test
+    public void satisfiesInterface() {
+        double[][] original = createTestData();
+        double[][] result = preprocessor.preprocessSimilarities(original);
+        validPreprocessing(original, result, preprocessor::originalIndexOf);
+    }
+
+    @Test
+    public void mayDecreaseOnly() {
+        double[][] original = createTestData();
+        double[][] result = preprocessor.preprocessSimilarities(original);
+        withAllValues(preprocessor, original, result, (originalValue, preprocessed) -> {
+            if (originalValue == 0.0) {
+                assertEquals(0.0, preprocessed.orElse(0.0), EPSILON);
+            } else {
+                assertTrue(preprocessed.get() <= originalValue);
+            }
+        });
+    }
+
+}

jplag/src/test/java/de/jplag/clustering/preprocessors/PercentilePreprocessorTest.java

@@ -0,0 +1,43 @@
+package de.jplag.clustering.preprocessors;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Optional;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class PercentilePreprocessorTest extends PreprocessingTestBase {
+
+    private static final double EPSILON = 0.0000001;
+
+    PercentileThresholdProcessor preprocessor;
+
+    @Before
+    public void init() {
+        preprocessor = new PercentileThresholdProcessor(0.5f);
+    }
+
+    @Test
+    public void satisfiesInterface() {
+        double[][] original = createTestData();
+        double[][] result = preprocessor.preprocessSimilarities(original);
+        validPreprocessing(original, result, preprocessor::originalIndexOf);
+    }
+
+    @Test
+    public void removedBelowPercentile() {
+        double[][] original = createTestData();
+        double[][] result = preprocessor.preprocessSimilarities(original);
+        withAllValues(preprocessor, original, result, (originalValue, preprocessed) -> {
+            // Values are only 0.1 and 0.5; percentile 0.5 should only preserve values of
+            // 0.5.
+            if (originalValue > 0.1) {
+                assertEquals(Optional.of(originalValue), preprocessed);
+            } else {
+                assertEquals(0.0, preprocessed.orElse(0.0), EPSILON);
+            }
+        });
+    }
+
+}

jplag/src/test/java/de/jplag/clustering/preprocessors/PreprocessingTestBase.java

@@ -0,0 +1,89 @@
+package de.jplag.clustering.preprocessors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.BiConsumer;
+import java.util.function.IntUnaryOperator;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.linear.ArrayRealVector;
+import org.apache.commons.math3.linear.RealMatrix;
+
+import de.jplag.clustering.ClusteringPreprocessor;
+
+public class PreprocessingTestBase {
+
+    private static final double EPSILON = 0.000001;
+
+    public double[][] createTestData() {
+        RealMatrix similarity = new Array2DRowRealMatrix(5, 5);
+        for (int i = 0; i < 4; i++) {
+            similarity.setEntry(i, i, 1);
+        }
+        // These are similar
+        setEntries(similarity, 0, 1, 0.5);
+        setEntries(similarity, 2, 3, 0.5f);
+
+        // Others are dissimilar
+        setEntries(similarity, 0, 2, 0.1f);
+        setEntries(similarity, 0, 3, 0.1f);
+        setEntries(similarity, 1, 2, 0.1f);
+        setEntries(similarity, 1, 3, 0.1f);
+
+        // last row is empty
+
+        return similarity.getData();
+    }
+
+    public void validPreprocessing(double[][] originalArray, double[][] resultArray, IntUnaryOperator originalIndex) {
+        RealMatrix result = new Array2DRowRealMatrix(resultArray, false);
+        RealMatrix original = new Array2DRowRealMatrix(originalArray, false);
+        assertEquals("not a square matrix", result.getColumnDimension(), result.getRowDimension());
+
+        List<Integer> usedOriginalIndices = IntStream.range(0, result.getColumnDimension()).map(originalIndex).boxed().collect(Collectors.toList());
+
+        assertEquals("original indices not unique", usedOriginalIndices.size(), new HashSet<>(usedOriginalIndices).size());
+
+        assertTrue("original indices valid", usedOriginalIndices.stream().allMatch(index -> index >= 0 && index < original.getColumnDimension()));
+
+        double[] columnNorms = IntStream.range(0, result.getColumnDimension()).mapToObj(index -> result.getColumn(index))
+                .mapToDouble(array -> new ArrayRealVector(array, false).getNorm()).toArray();
+
+        for (int i = 0; i < columnNorms.length; i++) {
+            assertTrue("produced zero column in column " + i, columnNorms[i] > EPSILON);
+        }
+    }
+
+    public double[][] withAllValues(ClusteringPreprocessor preprocessor, double[][] original, double[][] result,
+            BiConsumer<Double, Optional<Double>> originalAndPreprocessedConsumer) {
+
+        // Construct mapping from original indices to preprocessed indices as implied by
+        // originalIndexOf
+        Map<Integer, Integer> mappedIndices = IntStream.range(0, result.length).collect(HashMap::new,
+                (map, index) -> map.put(preprocessor.originalIndexOf(index), index), HashMap::putAll);
+        for (int j = 0; j < original.length; j++) {
+            double[] row = original[j];
+            Optional<double[]> mappedRow = Optional.ofNullable(mappedIndices.get(j)).map(index -> result[index]);
+            for (int i = 0; i < row.length; i++) {
+                double originalValue = row[i];
+                Optional<Double> value = Optional.ofNullable(mappedIndices.get(i)).flatMap(index -> mappedRow.map(x -> x[index]));
+                originalAndPreprocessedConsumer.accept(originalValue, value);
+            }
+        }
+
+        return result;
+    }
+
+    private static void setEntries(RealMatrix matrix, int i, int j, double value) {
+        matrix.setEntry(i, j, value);
+        matrix.setEntry(j, i, value);
+    }
+}