Merge branch 'master' into refactoring/languages

# Conflicts:
#	core/src/main/java/de/jplag/SubsequenceHashLookupTable.java
#	languages.api/src/main/java/de/jplag/TokenHashMap.java
#	languages.api/src/main/java/de/jplag/TokenList.java
#	languages/text/pom.xml
#	languages/text/src/main/antlr/text.g
#	languages/text/src/main/java/de/jplag/text/AntlrParserToken.java
#	languages/text/src/main/java/de/jplag/text/InputState.java

README.md

@@ -29,7 +29,7 @@ In the following, a list of all supported languages with their supported languag
 | [Scheme](http://www.scheme-reports.org)                          |       ? | scheme                | unknown | JavaCC |
 | [EMF Metamodel](https://www.eclipse.org/modeling/emf/)           |  2.25.0 | emf-metamodel         | alpha | EMF |
 | [EMF Metamodel](https://www.eclipse.org/modeling/emf/) (dynamic) |  2.25.0 | emf-metamodel-dynamic | alpha | EMF |
-| Text (naive)                                                     |       - | text                  | legacy | ANTLR |
+| Text (naive)                                                     |       - | text                  | legacy | CoreNLP |
 
 ## Download and Installation
 
@@ -98,8 +98,8 @@ JPlagResult result = jplag.run();
 List<JPlagComparison> comparisons = result.getComparisons();
 
 // Optional
-File outputDir = new File("/path/to/output");
-Report report = new Report(outputDir);
+ReportObjectFactory reportObjectFactory = new ReportObjectFactory();
+reportObjectFactory.createAndSaveReport(result, "/path/to/output");
 
 report.writeResult(result);
 ```

cli/src/main/java/de/jplag/CLI.java

@@ -1,11 +1,39 @@
 package de.jplag;
 
-import static de.jplag.CommandLineArgument.*;
+import static de.jplag.CommandLineArgument.BASE_CODE;
+import static de.jplag.CommandLineArgument.CLUSTER_AGGLOMERATIVE_INTER_CLUSTER_SIMILARITY;
+import static de.jplag.CommandLineArgument.CLUSTER_AGGLOMERATIVE_THRESHOLD;
+import static de.jplag.CommandLineArgument.CLUSTER_ALGORITHM;
+import static de.jplag.CommandLineArgument.CLUSTER_DISABLE;
+import static de.jplag.CommandLineArgument.CLUSTER_METRIC;
+import static de.jplag.CommandLineArgument.CLUSTER_PREPROCESSING_CDF;
+import static de.jplag.CommandLineArgument.CLUSTER_PREPROCESSING_NONE;
+import static de.jplag.CommandLineArgument.CLUSTER_PREPROCESSING_PERCENTILE;
+import static de.jplag.CommandLineArgument.CLUSTER_PREPROCESSING_THRESHOLD;
+import static de.jplag.CommandLineArgument.CLUSTER_SPECTRAL_BANDWIDTH;
+import static de.jplag.CommandLineArgument.CLUSTER_SPECTRAL_KMEANS_ITERATIONS;
+import static de.jplag.CommandLineArgument.CLUSTER_SPECTRAL_MAX_RUNS;
+import static de.jplag.CommandLineArgument.CLUSTER_SPECTRAL_MIN_RUNS;
+import static de.jplag.CommandLineArgument.CLUSTER_SPECTRAL_NOISE;
+import static de.jplag.CommandLineArgument.COMPARISON_MODE;
+import static de.jplag.CommandLineArgument.DEBUG;
+import static de.jplag.CommandLineArgument.EXCLUDE_FILE;
+import static de.jplag.CommandLineArgument.LANGUAGE;
+import static de.jplag.CommandLineArgument.MIN_TOKEN_MATCH;
+import static de.jplag.CommandLineArgument.NEW_DIRECTORY;
+import static de.jplag.CommandLineArgument.OLD_DIRECTORY;
+import static de.jplag.CommandLineArgument.RESULT_FOLDER;
+import static de.jplag.CommandLineArgument.ROOT_DIRECTORY;
+import static de.jplag.CommandLineArgument.SHOWN_COMPARISONS;
+import static de.jplag.CommandLineArgument.SIMILARITY_THRESHOLD;
+import static de.jplag.CommandLineArgument.SUBDIRECTORY;
+import static de.jplag.CommandLineArgument.SUFFIXES;
+import static de.jplag.CommandLineArgument.VERBOSITY;
 
 import java.security.SecureRandom;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
-import java.util.Optional;
 import java.util.Random;
 
 import net.sourceforge.argparse4j.ArgumentParsers;
@@ -17,14 +45,11 @@ import org.slf4j.ILoggerFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import de.jplag.clustering.ClusteringAlgorithm;
 import de.jplag.clustering.ClusteringOptions;
 import de.jplag.clustering.Preprocessing;
-import de.jplag.clustering.algorithm.InterClusterSimilarity;
 import de.jplag.exceptions.ExitException;
 import de.jplag.logger.CollectedLoggerFactory;
 import de.jplag.options.JPlagOptions;
-import de.jplag.options.SimilarityMetric;
 import de.jplag.options.Verbosity;
 import de.jplag.reporting.reportobject.ReportObjectFactory;
 import de.jplag.strategy.ComparisonMode;
@@ -125,53 +150,74 @@ public final class CLI {
         addAllMultiValueArgument(NEW_DIRECTORY.getListFrom(namespace), submissionDirectories);
         addAllMultiValueArgument(OLD_DIRECTORY.getListFrom(namespace), oldSubmissionDirectories);
 
-        JPlagOptions options = new JPlagOptions(submissionDirectories, oldSubmissionDirectories, LANGUAGE.getFrom(namespace));
-        options.setBaseCodeSubmissionName(BASE_CODE.getFrom(namespace));
-        options.setVerbosity(Verbosity.fromOption(VERBOSITY.getFrom(namespace)));
-        options.setDebugParser(DEBUG.getFrom(namespace));
-        options.setSubdirectoryName(SUBDIRECTORY.getFrom(namespace));
-        options.setFileSuffixes(fileSuffixes);
-        options.setExclusionFileName(EXCLUDE_FILE.getFrom(namespace));
-        options.setMinimumTokenMatch(MIN_TOKEN_MATCH.getFrom(namespace));
-        options.setSimilarityThreshold(SIMILARITY_THRESHOLD.getFrom(namespace));
-        options.setMaximumNumberOfComparisons(SHOWN_COMPARISONS.getFrom(namespace));
-        ComparisonMode.fromName(COMPARISON_MODE.getFrom(namespace)).ifPresentOrElse(options::setComparisonMode,
-                () -> logger.warn("Unknown comparison mode, using default mode!"));
-
-        ClusteringOptions.Builder clusteringBuilder = new ClusteringOptions.Builder();
-        Optional.ofNullable(!(Boolean) CLUSTER_DISABLE.getFrom(namespace)).ifPresent(clusteringBuilder::enabled);
-        Optional.ofNullable((ClusteringAlgorithm) CLUSTER_ALGORITHM.getFrom(namespace)).ifPresent(clusteringBuilder::algorithm);
-        Optional.ofNullable((SimilarityMetric) CLUSTER_METRIC.getFrom(namespace)).ifPresent(clusteringBuilder::similarityMetric);
-        Optional.ofNullable((Float) CLUSTER_SPECTRAL_BANDWIDTH.getFrom(namespace)).ifPresent(clusteringBuilder::spectralKernelBandwidth);
-        Optional.ofNullable((Float) CLUSTER_SPECTRAL_NOISE.getFrom(namespace)).ifPresent(clusteringBuilder::spectralGaussianProcessVariance);
-        Optional.ofNullable((Integer) CLUSTER_SPECTRAL_MIN_RUNS.getFrom(namespace)).ifPresent(clusteringBuilder::spectralMinRuns);
-        Optional.ofNullable((Integer) CLUSTER_SPECTRAL_MAX_RUNS.getFrom(namespace)).ifPresent(clusteringBuilder::spectralMaxRuns);
-        Optional.ofNullable((Integer) CLUSTER_SPECTRAL_KMEANS_ITERATIONS.getFrom(namespace))
-                .ifPresent(clusteringBuilder::spectralMaxKMeansIterationPerRun);
-        Optional.ofNullable((Float) CLUSTER_AGGLOMERATIVE_THRESHOLD.getFrom(namespace)).ifPresent(clusteringBuilder::agglomerativeThreshold);
-        Optional.ofNullable((InterClusterSimilarity) CLUSTER_AGGLOMERATIVE_INTER_CLUSTER_SIMILARITY.getFrom(namespace))
-                .ifPresent(clusteringBuilder::agglomerativeInterClusterSimilarity);
-        Optional.ofNullable((Boolean) CLUSTER_PREPROCESSING_NONE.getFrom(namespace)).ifPresent(none -> {
-            if (none) {
-                clusteringBuilder.preprocessor(Preprocessing.NONE);
+        var language = LanguageLoader.getLanguage(LANGUAGE.getFrom(namespace)).orElseThrow();
+        var comparisonModeOptional = ComparisonMode.fromName(COMPARISON_MODE.getFrom(namespace));
+        if (comparisonModeOptional.isEmpty()) {
+            logger.warn("Unknown comparison mode, using default mode!");
+        }
+        var comparisonMode = comparisonModeOptional.orElse(JPlagOptions.DEFAULT_COMPARISON_MODE);
+
+        ClusteringOptions clusteringOptions = getClusteringOptions(namespace);
+
+        return new JPlagOptions(language, MIN_TOKEN_MATCH.getFrom(namespace), submissionDirectories, oldSubmissionDirectories,
+                BASE_CODE.getFrom(namespace), SUBDIRECTORY.getFrom(namespace), Arrays.stream(fileSuffixes).toList(), EXCLUDE_FILE.getFrom(namespace),
+                JPlagOptions.DEFAULT_SIMILARITY_METRIC, SIMILARITY_THRESHOLD.getFrom(namespace), SHOWN_COMPARISONS.getFrom(namespace),
+                clusteringOptions, comparisonMode, Verbosity.fromOption(VERBOSITY.getFrom(namespace)), DEBUG.getFrom(namespace));
+    }
+
+    private static ClusteringOptions getClusteringOptions(Namespace namespace) {
+        ClusteringOptions clusteringOptions = new ClusteringOptions();
+        if (CLUSTER_DISABLE.isSet(namespace)) {
+            boolean disabled = CLUSTER_DISABLE.getFrom(namespace);
+            clusteringOptions = clusteringOptions.withEnabled(!disabled);
+        }
+        if (CLUSTER_ALGORITHM.isSet(namespace)) {
+            clusteringOptions = clusteringOptions.withAlgorithm(CLUSTER_ALGORITHM.getFrom(namespace));
+        }
+        if (CLUSTER_METRIC.isSet(namespace)) {
+            clusteringOptions = clusteringOptions.withSimilarityMetric(CLUSTER_METRIC.getFrom(namespace));
+        }
+        if (CLUSTER_SPECTRAL_BANDWIDTH.isSet(namespace)) {
+            clusteringOptions = clusteringOptions.withSpectralKernelBandwidth(CLUSTER_SPECTRAL_BANDWIDTH.getFrom(namespace));
+        }
+        if (CLUSTER_SPECTRAL_NOISE.isSet(namespace)) {
+            clusteringOptions = clusteringOptions.withSpectralGaussianProcessVariance(CLUSTER_SPECTRAL_NOISE.getFrom(namespace));
+        }
+        if (CLUSTER_SPECTRAL_MIN_RUNS.isSet(namespace)) {
+            clusteringOptions = clusteringOptions.withSpectralMinRuns(CLUSTER_SPECTRAL_MIN_RUNS.getFrom(namespace));
+        }
+        if (CLUSTER_SPECTRAL_MAX_RUNS.isSet(namespace)) {
+            clusteringOptions = clusteringOptions.withSpectralMaxRuns(CLUSTER_SPECTRAL_MAX_RUNS.getFrom(namespace));
+        }
+        if (CLUSTER_SPECTRAL_KMEANS_ITERATIONS.isSet(namespace)) {
+            clusteringOptions = clusteringOptions.withSpectralMaxKMeansIterationPerRun(CLUSTER_SPECTRAL_KMEANS_ITERATIONS.getFrom(namespace));
+        }
+        if (CLUSTER_AGGLOMERATIVE_THRESHOLD.isSet(namespace)) {
+            clusteringOptions = clusteringOptions.withAgglomerativeThreshold(CLUSTER_AGGLOMERATIVE_THRESHOLD.getFrom(namespace));
+        }
+        if (CLUSTER_AGGLOMERATIVE_INTER_CLUSTER_SIMILARITY.isSet(namespace)) {
+            clusteringOptions = clusteringOptions
+                    .withAgglomerativeInterClusterSimilarity(CLUSTER_AGGLOMERATIVE_INTER_CLUSTER_SIMILARITY.getFrom(namespace));
+        }
+        if (CLUSTER_PREPROCESSING_NONE.isSet(namespace)) {
+            if (CLUSTER_PREPROCESSING_NONE.getFrom(namespace)) {
+                clusteringOptions = clusteringOptions.withPreprocessor(Preprocessing.NONE);
             }
-        });
-        Optional.ofNullable((Boolean) CLUSTER_PREPROCESSING_CDF.getFrom(namespace)).ifPresent(cdf -> {
-            if (cdf) {
-                clusteringBuilder.preprocessor(Preprocessing.CUMULATIVE_DISTRIBUTION_FUNCTION);
+        }
+        if (CLUSTER_PREPROCESSING_CDF.isSet(namespace)) {
+            if (CLUSTER_PREPROCESSING_CDF.getFrom(namespace)) {
+                clusteringOptions = clusteringOptions.withPreprocessor(Preprocessing.CUMULATIVE_DISTRIBUTION_FUNCTION);
             }
-        });
-        Optional.ofNullable((Float) CLUSTER_PREPROCESSING_PERCENTILE.getFrom(namespace)).ifPresent(percentile -> {
-            clusteringBuilder.preprocessor(Preprocessing.PERCENTILE);
-            clusteringBuilder.preprocessorPercentile(percentile);
-        });
-        Optional.ofNullable((Float) CLUSTER_PREPROCESSING_THRESHOLD.getFrom(namespace)).ifPresent(threshold -> {
-            clusteringBuilder.preprocessor(Preprocessing.THRESHOLD);
-            clusteringBuilder.preprocessorPercentile(threshold);
-        });
-        options.setClusteringOptions(clusteringBuilder.build());
-
-        return options;
+        }
+        if (CLUSTER_PREPROCESSING_PERCENTILE.isSet(namespace)) {
+            clusteringOptions = clusteringOptions.withPreprocessor(Preprocessing.PERCENTILE)
+                    .withPreprocessorPercentile(CLUSTER_PREPROCESSING_PERCENTILE.getFrom(namespace));
+        }
+        if (CLUSTER_PREPROCESSING_THRESHOLD.isSet(namespace)) {
+            clusteringOptions = clusteringOptions.withPreprocessor(Preprocessing.THRESHOLD)
+                    .withPreprocessorPercentile(CLUSTER_PREPROCESSING_THRESHOLD.getFrom(namespace));
+        }
+        return clusteringOptions;
     }
 
     private String generateDescription() {

cli/src/main/java/de/jplag/CommandLineArgument.java

@@ -48,42 +48,40 @@ public enum CommandLineArgument {
     SUFFIXES(new Builder("-p", String.class).argumentGroup(ADVANCED_GROUP)),
     EXCLUDE_FILE(new Builder("-x", String.class).argumentGroup(ADVANCED_GROUP)),
     MIN_TOKEN_MATCH("-t", Integer.class),
-    SIMILARITY_THRESHOLD(new Builder("-m", Float.class).defaultsTo(DEFAULT_SIMILARITY_THRESHOLD).argumentGroup(ADVANCED_GROUP)),
+    SIMILARITY_THRESHOLD(new Builder("-m", Double.class).defaultsTo(DEFAULT_SIMILARITY_THRESHOLD).argumentGroup(ADVANCED_GROUP)),
     SHOWN_COMPARISONS(new Builder("-n", Integer.class).defaultsTo(DEFAULT_SHOWN_COMPARISONS)),
     RESULT_FOLDER(new Builder("-r", String.class).defaultsTo("result")),
     COMPARISON_MODE(new Builder("-c", String.class).defaultsTo(DEFAULT_COMPARISON_MODE.getName()).choices(ComparisonMode.allNames())),
     CLUSTER_DISABLE(new Builder("--cluster-skip", Boolean.class).argumentGroup(CLUSTERING_GROUP_NAME).action(Arguments.storeTrue())),
     CLUSTER_ALGORITHM(
             new Builder("--cluster-alg", ClusteringAlgorithm.class).argumentGroup(CLUSTERING_GROUP_NAME)
-                    .defaultsTo(ClusteringOptions.DEFAULTS.getAlgorithm())),
+                    .defaultsTo(new ClusteringOptions().algorithm())),
     CLUSTER_METRIC(
             new Builder("--cluster-metric", SimilarityMetric.class).argumentGroup(CLUSTERING_GROUP_NAME)
-                    .defaultsTo(ClusteringOptions.DEFAULTS.getSimilarityMetric())),
+                    .defaultsTo(new ClusteringOptions().similarityMetric())),
     CLUSTER_SPECTRAL_BANDWIDTH(
-            new Builder("--cluster-spectral-bandwidth", Float.class).metaVar("bandwidth")
-                    .defaultsTo(ClusteringOptions.DEFAULTS.getSpectralKernelBandwidth()).hidden()),
+            new Builder("--cluster-spectral-bandwidth", Double.class).metaVar("bandwidth")
+                    .defaultsTo(new ClusteringOptions().spectralKernelBandwidth()).hidden()),
     CLUSTER_SPECTRAL_NOISE(
-            new Builder("--cluster-spectral-noise", Float.class).metaVar("noise")
-                    .defaultsTo(ClusteringOptions.DEFAULTS.getSpectralGaussianProcessVariance()).hidden()),
+            new Builder("--cluster-spectral-noise", Double.class).metaVar("noise")
+                    .defaultsTo(new ClusteringOptions().spectralGaussianProcessVariance()).hidden()),
     CLUSTER_SPECTRAL_MIN_RUNS(
-            new Builder("--cluster-spectral-min-runs", Integer.class).metaVar("min").defaultsTo(ClusteringOptions.DEFAULTS.getSpectralMinRuns())
-                    .hidden()),
+            new Builder("--cluster-spectral-min-runs", Integer.class).metaVar("min").defaultsTo(new ClusteringOptions().spectralMinRuns()).hidden()),
     CLUSTER_SPECTRAL_MAX_RUNS(
-            new Builder("--cluster-spectral-max-runs", Integer.class).metaVar("max").defaultsTo(ClusteringOptions.DEFAULTS.getSpectralMaxRuns())
-                    .hidden()),
+            new Builder("--cluster-spectral-max-runs", Integer.class).metaVar("max").defaultsTo(new ClusteringOptions().spectralMaxRuns()).hidden()),
     CLUSTER_SPECTRAL_KMEANS_ITERATIONS(
             new Builder("--cluster-spectral-kmeans-interations", Integer.class).metaVar("iterations")
-                    .defaultsTo(ClusteringOptions.DEFAULTS.getSpectralMaxKMeansIterationPerRun()).hidden()),
+                    .defaultsTo(new ClusteringOptions().spectralMaxKMeansIterationPerRun()).hidden()),
     CLUSTER_AGGLOMERATIVE_THRESHOLD(
-            new Builder("--cluster-agglomerative-threshold", Float.class).metaVar("threshold")
-                    .defaultsTo(ClusteringOptions.DEFAULTS.getAgglomerativeThreshold()).hidden()),
+            new Builder("--cluster-agglomerative-threshold", Double.class).metaVar("threshold")
+                    .defaultsTo(new ClusteringOptions().agglomerativeThreshold()).hidden()),
     CLUSTER_AGGLOMERATIVE_INTER_CLUSTER_SIMILARITY(
             new Builder("--cluster-agglomerative-inter-cluster-similarity", InterClusterSimilarity.class)
-                    .defaultsTo(ClusteringOptions.DEFAULTS.getAgglomerativeInterClusterSimilarity()).hidden()),
+                    .defaultsTo(new ClusteringOptions().agglomerativeInterClusterSimilarity()).hidden()),
     CLUSTER_PREPROCESSING_NONE(new Builder("--cluster-pp-none", Boolean.class).action(Arguments.storeTrue()).hidden()),
     CLUSTER_PREPROCESSING_CDF(new Builder("--cluster-pp-cdf", Boolean.class).action(Arguments.storeTrue()).hidden()),
-    CLUSTER_PREPROCESSING_PERCENTILE(new Builder("--cluster-pp-percentile", Float.class).metaVar("percentile").hidden()),
-    CLUSTER_PREPROCESSING_THRESHOLD(new Builder("--cluster-pp-threshold", Float.class).metaVar("threshold").hidden());
+    CLUSTER_PREPROCESSING_PERCENTILE(new Builder("--cluster-pp-percentile", Double.class).metaVar("percentile").hidden()),
+    CLUSTER_PREPROCESSING_THRESHOLD(new Builder("--cluster-pp-threshold", Double.class).metaVar("threshold").hidden());
 
     /**
      * The identifier of the default {@link Language}.
@@ -147,6 +145,15 @@ public enum CommandLineArgument {
         return namespace.get(flagWithoutDash());
     }
 
+    /**
+     * Returns whether the value of this argument is set to a value not equal to {@code null}.
+     * @param namespace stores a value for the argument
+     * @return the indicator
+     */
+    public boolean isSet(Namespace namespace) {
+        return namespace.get(flagWithoutDash()) != null;
+    }
+
     /**
      * Returns the value of this argument for arguments that allow more than a single value. Convenience method for
      * {@link Namespace#getList(String)} and {@link CommandLineArgument#flagWithoutDash()}.

cli/src/test/java/de/jplag/cli/BaseCodeOptionTest.java

@@ -1,8 +1,7 @@
 package de.jplag.cli;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import java.util.Optional;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 import org.junit.jupiter.api.Test;
 
@@ -15,13 +14,13 @@ class BaseCodeOptionTest extends CommandLineInterfaceTest {
     @Test
     void testDefaultValue() {
         buildOptionsFromCLI(CURRENT_DIRECTORY);
-        assertEquals(Optional.empty(), options.getBaseCodeSubmissionName());
+        assertNull(options.baseCodeSubmissionName());
     }
 
     @Test
     void testCustomName() {
         String argument = buildArgument(CommandLineArgument.BASE_CODE, NAME);
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(NAME, options.getBaseCodeSubmissionName().get());
+        assertEquals(NAME, options.baseCodeSubmissionName());
     }
 }

cli/src/test/java/de/jplag/cli/ClusteringTest.java

@@ -15,35 +15,35 @@ class ClusteringTest extends CommandLineInterfaceTest {
     void parseSkipClustering() {
         String argument = CommandLineArgument.CLUSTER_DISABLE.flag();
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(false, options.getClusteringOptions().isEnabled());
+        assertEquals(false, options.clusteringOptions().enabled());
     }
 
     @Test
     void parseDefaultClustering() {
         buildOptionsFromCLI(CURRENT_DIRECTORY);
-        assertEquals(true, options.getClusteringOptions().isEnabled());
+        assertEquals(true, options.clusteringOptions().enabled());
     }
 
     @Test
     void parsePercentilePreProcessor() {
-        String argument = buildArgument(CommandLineArgument.CLUSTER_PREPROCESSING_PERCENTILE, Float.toString(0.5f));
+        String argument = buildArgument(CommandLineArgument.CLUSTER_PREPROCESSING_PERCENTILE, Double.toString(0.5));
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(Preprocessing.PERCENTILE, options.getClusteringOptions().getPreprocessor());
-        assertEquals(0.5, options.getClusteringOptions().getPreprocessorPercentile(), EPSILON);
+        assertEquals(Preprocessing.PERCENTILE, options.clusteringOptions().preprocessor());
+        assertEquals(0.5, options.clusteringOptions().preprocessorPercentile(), EPSILON);
     }
 
     @Test
     void parseCdfPreProcessor() {
         String argument = CommandLineArgument.CLUSTER_PREPROCESSING_CDF.flag();
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(Preprocessing.CUMULATIVE_DISTRIBUTION_FUNCTION, options.getClusteringOptions().getPreprocessor());
+        assertEquals(Preprocessing.CUMULATIVE_DISTRIBUTION_FUNCTION, options.clusteringOptions().preprocessor());
     }
 
     @Test
     void parseNoPreProcessor() {
         String argument = CommandLineArgument.CLUSTER_PREPROCESSING_NONE.flag();
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(Preprocessing.NONE, options.getClusteringOptions().getPreprocessor());
+        assertEquals(Preprocessing.NONE, options.clusteringOptions().preprocessor());
     }
 
 }

cli/src/test/java/de/jplag/cli/CommandLineInterfaceTest.java

@@ -19,7 +19,7 @@ import de.jplag.options.JPlagOptions;
  */
 public abstract class CommandLineInterfaceTest {
     protected static final String CURRENT_DIRECTORY = ".";
-    protected static final float DELTA = 0.0001f;
+    protected static final double DELTA = 1E-5;
 
     protected CLI cli;
     protected Namespace namespace;

cli/src/test/java/de/jplag/cli/ComparisonModeTest.java

@@ -14,7 +14,7 @@ class ComparisonModeTest extends CommandLineInterfaceTest {
     @Test
     void testDefaultMode() {
         buildOptionsFromCLI(CURRENT_DIRECTORY);
-        assertEquals(JPlagOptions.DEFAULT_COMPARISON_MODE, options.getComparisonMode());
+        assertEquals(JPlagOptions.DEFAULT_COMPARISON_MODE, options.comparisonMode());
     }
 
     @Test
@@ -29,7 +29,7 @@ class ComparisonModeTest extends CommandLineInterfaceTest {
         ComparisonMode mode = ComparisonMode.NORMAL;
         String argument = buildArgument(CommandLineArgument.COMPARISON_MODE, mode.getName());
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(mode, options.getComparisonMode());
+        assertEquals(mode, options.comparisonMode());
     }
 
     @Test
@@ -37,7 +37,7 @@ class ComparisonModeTest extends CommandLineInterfaceTest {
         ComparisonMode mode = ComparisonMode.PARALLEL;
         String argument = buildArgument(CommandLineArgument.COMPARISON_MODE, mode.getName());
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(mode, options.getComparisonMode());
+        assertEquals(mode, options.comparisonMode());
     }
 
 }

cli/src/test/java/de/jplag/cli/LanguageTest.java

@@ -3,6 +3,9 @@ package de.jplag.cli;
 import static com.github.stefanbirkner.systemlambda.SystemLambda.catchSystemExit;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import java.util.Arrays;
+import java.util.List;
+
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
@@ -15,7 +18,7 @@ class LanguageTest extends CommandLineInterfaceTest {
     @Test
     void testDefaultLanguage() {
         buildOptionsFromCLI(CURRENT_DIRECTORY);
-        assertEquals(CommandLineArgument.DEFAULT_LANGUAGE_IDENTIFIER, options.getLanguageIdentifier());
+        assertEquals(CommandLineArgument.DEFAULT_LANGUAGE_IDENTIFIER, options.language().getIdentifier());
     }
 
     @Test
@@ -36,8 +39,17 @@ class LanguageTest extends CommandLineInterfaceTest {
         for (Language language : LanguageLoader.getAllAvailableLanguages().values()) {
             String argument = buildArgument(CommandLineArgument.LANGUAGE, language.getIdentifier());
             buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-            assertEquals(language.getIdentifier(), options.getLanguageIdentifier());
+            assertEquals(language.getIdentifier(), options.language().getIdentifier());
+            assertEquals(Arrays.asList(language.suffixes()), options.fileSuffixes());
         }
     }
 
+    @Test
+    void testCustomSuffixes() {
+        List<String> suffixes = List.of("x", "y", "z");
+        String argument = buildArgument(CommandLineArgument.SUFFIXES, String.join(",", suffixes));
+        buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
+        assertEquals(suffixes, options.fileSuffixes());
+    }
+
 }

cli/src/test/java/de/jplag/cli/MinTokenMatchTest.java

@@ -6,51 +6,43 @@ import static org.junit.jupiter.api.Assertions.*;
 import org.junit.jupiter.api.Test;
 
 import de.jplag.CommandLineArgument;
-import de.jplag.JPlag;
 
-public class MinTokenMatchTest extends CommandLineInterfaceTest {
+class MinTokenMatchTest extends CommandLineInterfaceTest {
 
     @Test
-    public void testLanguageDefault() {
+    void testLanguageDefault() {
         // Language defaults not set yet:
         buildOptionsFromCLI(CURRENT_DIRECTORY);
-        assertNull(options.getMinimumTokenMatch());
-        assertNull(options.getLanguage());
-
-        // Init JPlag:
-        new JPlag(options);
-
-        // Now the language is set:
-        assertNotNull(options.getLanguage());
-        assertEquals(options.getLanguage().minimumTokenMatch(), options.getMinimumTokenMatch().intValue());
+        assertNotNull(options.language());
+        assertEquals(options.language().minimumTokenMatch(), options.minimumTokenMatch().intValue());
     }
 
     @Test
-    public void testInvalidInput() throws Exception {
+    void testInvalidInput() throws Exception {
         String argument = buildArgument(CommandLineArgument.MIN_TOKEN_MATCH, "Not an integer...");
         int statusCode = catchSystemExit(() -> buildOptionsFromCLI(argument, CURRENT_DIRECTORY));
         assertEquals(1, statusCode);
     }
 
     @Test
-    public void testUpperBound() throws Exception {
+    void testUpperBound() throws Exception {
         String argument = buildArgument(CommandLineArgument.MIN_TOKEN_MATCH, "2147483648"); // max value plus one
         int statusCode = catchSystemExit(() -> buildOptionsFromCLI(argument, CURRENT_DIRECTORY));
         assertEquals(1, statusCode);
     }
 
     @Test
-    public void testLowerBound() {
+    void testLowerBound() {
         String argument = buildArgument(CommandLineArgument.MIN_TOKEN_MATCH, Integer.toString(-1));
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(1, options.getMinimumTokenMatch().intValue());
+        assertEquals(1, options.minimumTokenMatch().intValue());
     }
 
     @Test
-    public void testValidThreshold() {
+    void testValidThreshold() {
         int expectedValue = 50;
         String argument = buildArgument(CommandLineArgument.MIN_TOKEN_MATCH, Integer.toString(expectedValue));
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(expectedValue, options.getMinimumTokenMatch().intValue());
+        assertEquals(expectedValue, options.minimumTokenMatch().intValue());
     }
 }

cli/src/test/java/de/jplag/cli/OldNewRootDirectoriesArgumentTest.java

@@ -4,52 +4,52 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import org.junit.jupiter.api.Test;
 
-public class OldNewRootDirectoriesArgumentTest extends CommandLineInterfaceTest {
+class OldNewRootDirectoriesArgumentTest extends CommandLineInterfaceTest {
     @Test
-    public void testNoRootDirectories() {
+    void testNoRootDirectories() {
         buildOptionsFromCLI();
 
-        assertEquals(0, options.getSubmissionDirectories().size());
-        assertEquals(0, options.getOldSubmissionDirectories().size());
+        assertEquals(0, options.submissionDirectories().size());
+        assertEquals(0, options.oldSubmissionDirectories().size());
     }
 
     @Test
-    public void testTwoRootDirectoryArguments() {
+    void testTwoRootDirectoryArguments() {
         buildOptionsFromCLI("root1", "root2");
 
-        assertEquals(2, options.getSubmissionDirectories().size());
-        assertEquals(0, options.getOldSubmissionDirectories().size());
+        assertEquals(2, options.submissionDirectories().size());
+        assertEquals(0, options.oldSubmissionDirectories().size());
     }
 
     @Test
-    public void testNewOption() {
+    void testNewOption() {
         buildOptionsFromCLI("-new", "root1", "root2");
 
-        assertEquals(2, options.getSubmissionDirectories().size());
-        assertEquals(0, options.getOldSubmissionDirectories().size());
+        assertEquals(2, options.submissionDirectories().size());
+        assertEquals(0, options.oldSubmissionDirectories().size());
     }
 
     @Test
-    public void testDoubleNewOption() {
+    void testDoubleNewOption() {
         buildOptionsFromCLI("-new", "root1", "-new", "root2");
 
-        assertEquals(2, options.getSubmissionDirectories().size());
-        assertEquals(0, options.getOldSubmissionDirectories().size());
+        assertEquals(2, options.submissionDirectories().size());
+        assertEquals(0, options.oldSubmissionDirectories().size());
     }
 
     @Test
-    public void testOldOption() {
+    void testOldOption() {
         buildOptionsFromCLI("-old", "root1");
 
-        assertEquals(0, options.getSubmissionDirectories().size());
-        assertEquals(1, options.getOldSubmissionDirectories().size());
+        assertEquals(0, options.submissionDirectories().size());
+        assertEquals(1, options.oldSubmissionDirectories().size());
     }
 
     @Test
-    public void testNewAndOldOption() {
+    void testNewAndOldOption() {
         buildOptionsFromCLI("-new", "root1", "-old", "root2");
 
-        assertEquals(1, options.getSubmissionDirectories().size());
-        assertEquals(1, options.getOldSubmissionDirectories().size());
+        assertEquals(1, options.submissionDirectories().size());
+        assertEquals(1, options.oldSubmissionDirectories().size());
     }
 }

cli/src/test/java/de/jplag/cli/SimiliarityThresholdTest.java

@@ -8,40 +8,40 @@ import org.junit.jupiter.api.Test;
 import de.jplag.CommandLineArgument;
 import de.jplag.options.JPlagOptions;
 
-public class SimiliarityThresholdTest extends CommandLineInterfaceTest {
+class SimiliarityThresholdTest extends CommandLineInterfaceTest {
 
     @Test
-    public void testDefaultThreshold() {
+    void testDefaultThreshold() {
         buildOptionsFromCLI(CURRENT_DIRECTORY);
-        assertEquals(JPlagOptions.DEFAULT_SIMILARITY_THRESHOLD, options.getSimilarityThreshold(), DELTA);
+        assertEquals(JPlagOptions.DEFAULT_SIMILARITY_THRESHOLD, options.similarityThreshold(), DELTA);
     }
 
     @Test
-    public void testInvalidThreshold() throws Exception {
-        String argument = buildArgument(CommandLineArgument.SIMILARITY_THRESHOLD, "Not a float...");
+    void testInvalidThreshold() throws Exception {
+        String argument = buildArgument(CommandLineArgument.SIMILARITY_THRESHOLD, "Not a Double...");
         int statusCode = catchSystemExit(() -> buildOptionsFromCLI(argument, CURRENT_DIRECTORY));
-        assertEquals(1, statusCode);
+        assertEquals(1.0, statusCode);
     }
 
     @Test
-    public void testLowerBound() {
-        String argument = buildArgument(CommandLineArgument.SIMILARITY_THRESHOLD, Float.toString(-1f));
+    void testLowerBound() {
+        String argument = buildArgument(CommandLineArgument.SIMILARITY_THRESHOLD, Double.toString(-1.0));
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(0f, options.getSimilarityThreshold(), DELTA);
+        assertEquals(0.0, options.similarityThreshold(), DELTA);
     }
 
     @Test
-    public void testUpperBound() {
-        String argument = buildArgument(CommandLineArgument.SIMILARITY_THRESHOLD, Float.toString(101f));
+    void testUpperBound() {
+        String argument = buildArgument(CommandLineArgument.SIMILARITY_THRESHOLD, Double.toString(101.0));
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(100f, options.getSimilarityThreshold(), DELTA);
+        assertEquals(100.0, options.similarityThreshold(), DELTA);
     }
 
     @Test
-    public void testValidThreshold() {
-        float expectedValue = 50f;
-        String argument = buildArgument(CommandLineArgument.SIMILARITY_THRESHOLD, Float.toString(expectedValue));
+    void testValidThreshold() {
+        double expectedValue = 50.0;
+        String argument = buildArgument(CommandLineArgument.SIMILARITY_THRESHOLD, Double.toString(expectedValue));
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(expectedValue, options.getSimilarityThreshold(), DELTA);
+        assertEquals(expectedValue, options.similarityThreshold(), DELTA);
     }
 }

cli/src/test/java/de/jplag/cli/StoredMatchesTest.java

@@ -8,39 +8,39 @@ import org.junit.jupiter.api.Test;
 import de.jplag.CommandLineArgument;
 import de.jplag.options.JPlagOptions;
 
-public class StoredMatchesTest extends CommandLineInterfaceTest {
+class StoredMatchesTest extends CommandLineInterfaceTest {
 
     @Test
-    public void testDefault() {
+    void testDefault() {
         buildOptionsFromCLI(CURRENT_DIRECTORY);
-        assertEquals(JPlagOptions.DEFAULT_SHOWN_COMPARISONS, options.getMaximumNumberOfComparisons());
+        assertEquals(JPlagOptions.DEFAULT_SHOWN_COMPARISONS, options.maximumNumberOfComparisons());
     }
 
     @Test
-    public void testValidThreshold() {
+    void testValidThreshold() {
         int expectedValue = 999;
         String argument = buildArgument(CommandLineArgument.SHOWN_COMPARISONS, Integer.toString(expectedValue));
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(expectedValue, options.getMaximumNumberOfComparisons());
+        assertEquals(expectedValue, options.maximumNumberOfComparisons());
     }
 
     @Test
-    public void testAll() {
-        int expectedValue = -1;
+    void testAll() {
+        int expectedValue = JPlagOptions.SHOW_ALL_COMPARISONS;
         String argument = buildArgument(CommandLineArgument.SHOWN_COMPARISONS, Integer.toString(expectedValue));
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(expectedValue, options.getMaximumNumberOfComparisons());
+        assertEquals(expectedValue, options.maximumNumberOfComparisons());
     }
 
     @Test
-    public void testLowerBound() {
+    void testLowerBound() {
         String argument = buildArgument(CommandLineArgument.SHOWN_COMPARISONS, Integer.toString(-2));
         buildOptionsFromCLI(argument, CURRENT_DIRECTORY);
-        assertEquals(-1, options.getMaximumNumberOfComparisons());
+        assertEquals(JPlagOptions.SHOW_ALL_COMPARISONS, options.maximumNumberOfComparisons());
     }
 
     @Test
-    public void testInvalidThreshold() throws Exception {
+    void testInvalidThreshold() throws Exception {
         String argument = buildArgument(CommandLineArgument.SHOWN_COMPARISONS, "Not an integer...");
         int statusCode = catchSystemExit(() -> buildOptionsFromCLI(argument, CURRENT_DIRECTORY));
         assertEquals(1, statusCode);

core/pom.xml

@@ -14,7 +14,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.13.3</version>
+            <version>2.13.4</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>

core/src/main/java/de/jplag/GreedyStringTiling.java

@@ -4,100 +4,69 @@ import static de.jplag.TokenConstants.FILE_END;
 import static de.jplag.TokenConstants.SEPARATOR_TOKEN;
 
 import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import de.jplag.options.JPlagOptions;
 
 /**
  * This class implements the Greedy String Tiling algorithm as introduced by Michael Wise. However, it is very specific
- * to the classes {@link TokenList}, {@link Token}, and {@link Match}. While this class was reworked, it still contains
- * some quirks from the initial version.
+ * to the classes {@link Token}, and {@link Match}.
  * @see <a href=
  * "https://www.researchgate.net/publication/262763983_String_Similarity_via_Greedy_String_Tiling_and_Running_Karp-Rabin_Matching">
  * String Similarity via Greedy String Tiling and Running Karp−Rabin Matching </a>
  */
 public class GreedyStringTiling {
 
-    private final JPlagOptions options;
+    private final int minimumMatchLength;
+    private final Map<Submission, SubsequenceHashLookupTable> cachedHashLookupTables = new IdentityHashMap<>();
+    private final Map<Submission, Set<Token>> baseCodeMarkings = new IdentityHashMap<>();
 
     public GreedyStringTiling(JPlagOptions options) {
-        this.options = options;
+        this.minimumMatchLength = options.minimumTokenMatch();
     }
 
     /**
-     * Creating hashes in linear time. The hash-code will be written in every Token for the next &lt;hashLength&gt; token
-     * (includes the Token itself).
-     * @param tokenList contains the tokens.
-     * @param hashLength is the hash length (condition: 1 &lt; hashLength &lt; 26)
-     * @param makeTable determines if a simple hash table is created in the structure.
+     * Compares the given submission with the base code submission. Marks the identified base code sections in the
+     * submission such that further comparisons do not generate matches for these parts. Must be called before generating a
+     * comparison with a regular submission for the given submission.
+     * @param submission is the submission to generate base-code markings for.
+     * @param baseCodeSubmission is the base code submission.
+     * @return the comparison of the submission with the base code submission.
      */
-    public void createHashes(TokenList tokenList, int hashLength, boolean makeTable) {
-        // Here the upper boundary of the hash length is set.
-        // It is determined by the number of bits of the 'int' data type and the number of tokens.
-        if (hashLength < 1) {
-            hashLength = 1;
+    public final JPlagComparison generateBaseCodeMarking(Submission submission, Submission baseCodeSubmission) {
+        JPlagComparison comparison = compare(submission, baseCodeSubmission);
+
+        List<Token> submissionTokenList = submission.getTokenList();
+        Set<Token> baseCodeMarking = new HashSet<>();
+        for (Match match : comparison.matches()) {
+            int startIndex = comparison.firstSubmission() == submission ? match.startOfFirst() : match.startOfSecond();
+            baseCodeMarking.addAll(submissionTokenList.subList(startIndex, startIndex + match.length()));
         }
-        hashLength = (hashLength < 26 ? hashLength : 25);
+        baseCodeMarkings.put(submission, baseCodeMarking);
 
-        if (tokenList.size() < hashLength) {
-            return;
-        }
+        // Remove the lookup table for the current submission to trigger a regeneration as hashes will change due to the new
+        // baseCodeMarking.
+        // This is a performance optimization to not suggest subsequences with baseCode for the matching.
+        // Removing this optimization would not change the result as the baseCode matches are additionally checked by validating
+        // that no match has a marked token (which baseCode-containing tokens are).
+        cachedHashLookupTables.remove(submission);
 
-        int modulo = ((1 << 6) - 1);   // Modulo 64!
-
-        int loops = tokenList.size() - hashLength;
-        tokenList.tokenHashes = (makeTable ? new TokenHashMap(3 * loops) : null);
-        int hash = 0;
-        int hashedLength = 0;
-        for (int i = 0; i < hashLength; i++) {
-            hash = (2 * hash) + (tokenList.getToken(i).type & modulo);
-            hashedLength++;
-            if (tokenList.getToken(i).isMarked()) {
-                hashedLength = 0;
-            }
-        }
-        int factor = (hashLength != 1 ? (2 << (hashLength - 2)) : 1);
-
-        if (makeTable) {
-            for (int i = 0; i < loops; i++) {
-                if (hashedLength >= hashLength) {
-                    tokenList.getToken(i).setHash(hash);
-                    tokenList.tokenHashes.put(hash, i);   // add into hashtable
-                } else {
-                    tokenList.getToken(i).setHash(-1);
-                }
-                hash -= factor * (tokenList.getToken(i).type & modulo);
-                hash = (2 * hash) + (tokenList.getToken(i + hashLength).type & modulo);
-                if (tokenList.getToken(i + hashLength).isMarked()) {
-                    hashedLength = 0;
-                } else {
-                    hashedLength++;
-                }
-            }
-        } else {
-            for (int i = 0; i < loops; i++) {
-                tokenList.getToken(i).setHash((hashedLength >= hashLength) ? hash : -1);
-                hash -= factor * (tokenList.getToken(i).type & modulo);
-                hash = (2 * hash) + (tokenList.getToken(i + hashLength).type & modulo);
-                if (tokenList.getToken(i + hashLength).isMarked()) {
-                    hashedLength = 0;
-                } else {
-                    hashedLength++;
-                }
-            }
-        }
-        tokenList.hashLength = hashLength;
+        return comparison;
     }
 
+    /**
+     * Compares the two submissions and generates matches between them. To exclude base code from the result, call
+     * {@link #generateBaseCodeMarking} with each submission beforehand.
+     * @param firstSubmission is one of the two submissions.
+     * @param secondSubmission is the other of the two submissions.
+     * @return the comparison between the two submissions.
+     */
     public final JPlagComparison compare(Submission firstSubmission, Submission secondSubmission) {
-        return swapAndCompare(firstSubmission, secondSubmission, false);
-    }
-
-    public final JPlagComparison compareWithBaseCode(Submission firstSubmission, Submission secondSubmission) {
-        return swapAndCompare(firstSubmission, secondSubmission, true);
-    }
-
-    private JPlagComparison swapAndCompare(Submission firstSubmission, Submission secondSubmission, boolean isBaseCodeComparison) {
         Submission smallerSubmission;
         Submission largerSubmission;
         if (firstSubmission.getTokenList().size() > secondSubmission.getTokenList().size()) {
@@ -107,129 +76,126 @@ public class GreedyStringTiling {
             smallerSubmission = firstSubmission;
             largerSubmission = secondSubmission;
         }
-        // if hashtable exists in first but not in second structure: flip around!
-        if (largerSubmission.getTokenList().tokenHashes == null && smallerSubmission.getTokenList().tokenHashes != null) {
-            Submission swap = smallerSubmission;
-            smallerSubmission = largerSubmission;
-            largerSubmission = swap;
-        }
-        return compare(smallerSubmission, largerSubmission, isBaseCodeComparison);
+        return compareInternal(smallerSubmission, largerSubmission);
     }
 
     /**
      * Compares two submissions. FILE_END is used as pivot
-     * @param firstSubmission is the submission with the smaller sequence.
-     * @param secondSubmission is the submission with the larger sequence.
-     * @param isBaseCodeComparison specifies whether one of the submissions is the base code.
+     * @param leftSubmission is the submission with the smaller sequence.
+     * @param rightSubmission is the submission with the larger sequence.
      * @return the comparison results.
      */
-    private JPlagComparison compare(Submission firstSubmission, Submission secondSubmission, boolean isBaseCodeComparison) {
-        // first and second refer to the list of tokens of the first and second submission:
-        TokenList first = firstSubmission.getTokenList();
-        TokenList second = secondSubmission.getTokenList();
+    private JPlagComparison compareInternal(Submission leftSubmission, Submission rightSubmission) {
+        List<Token> leftTokens = leftSubmission.getTokenList();
+        List<Token> rightTokens = rightSubmission.getTokenList();
 
-        // Initialize:
-        JPlagComparison comparison = new JPlagComparison(firstSubmission, secondSubmission);
-        int minimumTokenMatch = options.getMinimumTokenMatch(); // minimal required token match
-
-        if (first.size() <= minimumTokenMatch || second.size() <= minimumTokenMatch) { // <= because of pivots!
-            return comparison;
+        // comparison uses <= because it is assumed that the last token is a pivot (FILE_END)
+        if (leftTokens.size() <= minimumMatchLength || rightTokens.size() <= minimumMatchLength) {
+            return new JPlagComparison(leftSubmission, rightSubmission, List.of());
         }
 
-        markTokens(first, isBaseCodeComparison);
-        markTokens(second, isBaseCodeComparison);
+        Set<Token> leftMarkedTokens = initiallyMarkedTokens(leftSubmission);
+        Set<Token> rightMarkedTokens = initiallyMarkedTokens(rightSubmission);
 
-        // create hashes:
-        if (first.hashLength != minimumTokenMatch) {
-            createHashes(first, minimumTokenMatch, isBaseCodeComparison); // don't make table if it is not a base code comparison
-        }
-        if (second.hashLength != minimumTokenMatch || second.tokenHashes == null) {
-            createHashes(second, minimumTokenMatch, true);
-        }
+        SubsequenceHashLookupTable leftLookupTable = subsequenceHashLookupTableForSubmission(leftSubmission, leftMarkedTokens);
+        SubsequenceHashLookupTable rightLookupTable = subsequenceHashLookupTableForSubmission(rightSubmission, rightMarkedTokens);
 
-        List<Match> matches = new ArrayList<>();
-
-        // start the black magic:
-        int maxMatch;
+        int maximumMatchLength;
+        List<Match> globalMatches = new ArrayList<>();
         do {
-            maxMatch = minimumTokenMatch;
-            matches.clear();
-            for (int x = 0; x < first.size() - maxMatch; x++) {
-                List<Integer> hashedTokens = second.tokenHashes.get(first.getToken(x).getHash());
-                if (first.getToken(x).isMarked() || first.getToken(x).getHash() == -1) {
+            maximumMatchLength = minimumMatchLength;
+            List<Match> iterationMatches = new ArrayList<>();
+            for (int leftStartIndex = 0; leftStartIndex < leftTokens.size() - maximumMatchLength; leftStartIndex++) {
+                int leftSubsequenceHash = leftLookupTable.subsequenceHashForStartIndex(leftStartIndex);
+                if (leftMarkedTokens.contains(leftTokens.get(leftStartIndex)) || leftSubsequenceHash == SubsequenceHashLookupTable.NO_HASH) {
                     continue;
                 }
-                inner: for (Integer y : hashedTokens) {
-                    if (second.getToken(y).isMarked() || maxMatch >= second.size() - y) { // >= because of pivots!
+                List<Integer> possiblyMatchingRightStartIndexes = rightLookupTable
+                        .startIndexesOfPossiblyMatchingSubsequencesForSubsequenceHash(leftSubsequenceHash);
+                for (Integer rightStartIndex : possiblyMatchingRightStartIndexes) {
+                    // comparison uses >= because it is assumed that the last token is a pivot (FILE_END)
+                    if (rightMarkedTokens.contains(rightTokens.get(rightStartIndex)) || maximumMatchLength >= rightTokens.size() - rightStartIndex) {
                         continue;
                     }
 
-                    int j, hx, hy;
-                    for (j = maxMatch - 1; j >= 0; j--) { // begins comparison from behind
-                        if (first.getToken(hx = x + j).type != second.getToken(hy = y + j).type || first.getToken(hx).isMarked()
-                                || second.getToken(hy).isMarked()) {
-                            continue inner;
-                        }
+                    if (!subsequencesAreMatchingAndNotMarked(leftTokens.subList(leftStartIndex, leftStartIndex + maximumMatchLength),
+                            leftMarkedTokens, rightTokens.subList(rightStartIndex, rightStartIndex + maximumMatchLength), rightMarkedTokens)) {
+                        continue;
                     }
 
                     // expand match
-                    j = maxMatch;
-                    while (first.getToken(hx = x + j).type == second.getToken(hy = y + j).type && !first.getToken(hx).isMarked()
-                            && !second.getToken(hy).isMarked()) {
-                        j++;
+                    int offset = maximumMatchLength;
+                    while (leftTokens.get(leftStartIndex + offset).type == rightTokens.get(rightStartIndex + offset).type
+                            && !leftMarkedTokens.contains(leftTokens.get(leftStartIndex + offset))
+                            && !rightMarkedTokens.contains(rightTokens.get(rightStartIndex + offset))) {
+                        offset++;
                     }
 
-                    if (j > maxMatch && !isBaseCodeComparison || j != maxMatch && isBaseCodeComparison) {  // new biggest match? -> delete current
-                                                                                                           // smaller
-                        matches.clear();
-                        maxMatch = j;
+                    if (offset > maximumMatchLength) {
+                        iterationMatches.clear();
+                        maximumMatchLength = offset;
                     }
-                    addMatchIfNotOverlapping(matches, x, y, j);
+                    Match match = new Match(leftStartIndex, rightStartIndex, offset);
+                    addMatchIfNotOverlapping(iterationMatches, match);
                 }
             }
-            for (int i = matches.size() - 1; i >= 0; i--) {
-                int x = matches.get(i).startOfFirst();  // Beginning of/in sequence A
-                int y = matches.get(i).startOfSecond();  // Beginning of/in sequence B
-                comparison.addMatch(x, y, matches.get(i).length());
-                // in order that "Match" will be newly build (because reusing)
-                for (int j = matches.get(i).length(); j > 0; j--) {
-                    first.getToken(x).setMarked(true); // mark all Tokens!
-                    second.getToken(y).setMarked(true);
-                    if (isBaseCodeComparison) {
-                        first.getToken(x).setBasecode(true);
-                        second.getToken(y).setBasecode(true);
-                    }
-                    x++;
-                    y++;
+            for (Match match : iterationMatches) {
+                addMatchIfNotOverlapping(globalMatches, match);
+                int leftStartIndex = match.startOfFirst();
+                int rightStartIndex = match.startOfSecond();
+                for (int offset = 0; offset < match.length(); offset++) {
+                    leftMarkedTokens.add(leftTokens.get(leftStartIndex + offset));
+                    rightMarkedTokens.add(rightTokens.get(rightStartIndex + offset));
                 }
             }
+        } while (maximumMatchLength != minimumMatchLength);
+        return new JPlagComparison(leftSubmission, rightSubmission, globalMatches);
+    }
 
-        } while (maxMatch != minimumTokenMatch);
-
-        return comparison;
+    /**
+     * Checks if the two provided subsequences are equal and not marked. Comparison is performed backwards based on the
+     * assumption that the further tokens are away, the more likely they differ. leftTokens and rightTokens must be of equal
+     * size.
+     * @param leftTokens The subsequence of left tokens.
+     * @param leftMarkedTokens The marked tokens of the left token list.
+     * @param rightTokens The subsequence of right tokens.
+     * @param rightMarkedTokens The marked tokens of the right token list.
+     * @return true if the subsequences are matching and not marked, otherwise false.
+     */
+    private boolean subsequencesAreMatchingAndNotMarked(List<Token> leftTokens, Set<Token> leftMarkedTokens, List<Token> rightTokens,
+            Set<Token> rightMarkedTokens) {
+        for (int offset = leftTokens.size() - 1; offset >= 0; offset--) {
+            Token leftToken = leftTokens.get(offset);
+            Token rightToken = rightTokens.get(offset);
+            if (leftToken.type != rightToken.type || leftMarkedTokens.contains(leftToken) || rightMarkedTokens.contains(rightToken)) {
+                return false;
+            }
+        }
+        return true;
     }
 
-    private void addMatchIfNotOverlapping(List<Match> matches, int startA, int startB, int length) {
+    private void addMatchIfNotOverlapping(List<Match> matches, Match match) {
         for (int i = matches.size() - 1; i >= 0; i--) { // starting at the end is better(?)
-            if (matches.get(i).overlap(startA, startB, length)) {
+            if (matches.get(i).overlaps(match)) {
                 return; // no overlaps allowed!
             }
         }
-        matches.add(new Match(startA, startB, length));
+        matches.add(match);
     }
 
-    /**
-     * Disable finding a match at separator tokens and basecode matches for non-basecode comparisons.
-     * @param tokenList Tokens to mark.
-     * @param isBaseCodeComparison Whether the {@link Token#isBasecode()} matches should be enabled for matching.
-     */
-    private void markTokens(TokenList tokenList, boolean isBaseCodeComparison) {
-        for (Token token : tokenList.allTokens()) {
-            if (isBaseCodeComparison) {
-                token.setMarked(token.type == FILE_END || token.type == SEPARATOR_TOKEN);
-            } else {
-                token.setMarked(token.type == FILE_END || token.type == SEPARATOR_TOKEN || (token.isBasecode() && options.hasBaseCode()));
-            }
+    private Set<Token> initiallyMarkedTokens(Submission submission) {
+        Set<Token> baseCodeTokens = baseCodeMarkings.get(submission);
+        return submission.getTokenList().stream().filter(
+                token -> token.type == FILE_END || token.type == SEPARATOR_TOKEN || (baseCodeTokens != null && baseCodeTokens.contains(token)))
+                .collect(Collectors.toSet());
+    }
+
+    private SubsequenceHashLookupTable subsequenceHashLookupTableForSubmission(Submission submission, Set<Token> markedTokens) {
+        if (cachedHashLookupTables.containsKey(submission)) {
+            return cachedHashLookupTables.get(submission);
         }
+        SubsequenceHashLookupTable lookupTable = new SubsequenceHashLookupTable(minimumMatchLength, submission.getTokenList(), markedTokens);
+        cachedHashLookupTables.put(submission, lookupTable);
+        return lookupTable;
     }
 }

core/src/main/java/de/jplag/JPlag.java

@@ -1,15 +1,5 @@
 package de.jplag;
 
-import static de.jplag.options.Verbosity.LONG;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,7 +23,6 @@ public class JPlag {
     private final Language language;
     private final ComparisonStrategy comparisonStrategy;
     private final GreedyStringTiling coreAlgorithm; // Contains the comparison logic.
-    private final Set<String> excludedFileNames;
 
     /**
      * Creates and initializes a JPlag instance, parameterized by a set of options.
@@ -42,30 +31,8 @@ public class JPlag {
     public JPlag(JPlagOptions options) {
         this.options = options;
         coreAlgorithm = new GreedyStringTiling(options);
-        language = initializeLanguage(this.options);
-        comparisonStrategy = initializeComparisonStrategy(options.getComparisonMode());
-        excludedFileNames = Optional.ofNullable(this.options.getExclusionFileName()).map(this::readExclusionFile).orElse(Collections.emptySet());
-        options.setExcludedFiles(excludedFileNames); // store for report
-    }
-
-    /**
-     * If an exclusion file is given, it is read in and all strings are saved in the set "excluded".
-     * @param exclusionFileName the file name or path
-     */
-    private Set<String> readExclusionFile(final String exclusionFileName) {
-        try (BufferedReader reader = new BufferedReader(new FileReader(exclusionFileName, JPlagOptions.CHARSET))) {
-            final var excludedFileNames = reader.lines().collect(Collectors.toSet());
-            if (options.getVerbosity() == LONG) {
-                logger.info("Excluded files:");
-                for (var excludedFilename : excludedFileNames) {
-                    logger.info(excludedFilename);
-                }
-            }
-            return excludedFileNames;
-        } catch (IOException e) {
-            logger.error("Could not read exclusion file: " + e.getMessage(), e);
-            return Collections.emptySet();
-        }
+        language = this.options.language();
+        comparisonStrategy = initializeComparisonStrategy(options.comparisonMode());
     }
 
     /**
@@ -75,13 +42,9 @@ public class JPlag {
      */
     public JPlagResult run() throws ExitException {
         // Parse and validate submissions.
-        SubmissionSetBuilder builder = new SubmissionSetBuilder(language, options, excludedFileNames);
+        SubmissionSetBuilder builder = new SubmissionSetBuilder(language, options);
         SubmissionSet submissionSet = builder.buildSubmissionSet();
 
-        if (submissionSet.hasBaseCode()) {
-            coreAlgorithm.createHashes(submissionSet.getBaseCode().getTokenList(), options.getMinimumTokenMatch(), true);
-        }
-
         int submissionCount = submissionSet.numberOfSubmissions();
         if (submissionCount < 2) {
             throw new SubmissionException("Not enough valid submissions! (found " + submissionCount + " valid submissions)");
@@ -92,7 +55,7 @@ public class JPlag {
         if (logger.isInfoEnabled())
             logger.info("Total time for comparing submissions: {}", TimeUtil.formatDuration(result.getDuration()));
 
-        result.setClusteringResult(ClusteringFactory.getClusterings(result.getAllComparisons(), options.getClusteringOptions()));
+        result.setClusteringResult(ClusteringFactory.getClusterings(result.getAllComparisons(), options.clusteringOptions()));
 
         return result;
     }
@@ -103,21 +66,4 @@ public class JPlag {
             case PARALLEL -> new ParallelComparisonStrategy(options, coreAlgorithm);
         };
     }
-
-    private static Language initializeLanguage(JPlagOptions options) {
-        String languageIdentifier = options.getLanguageIdentifier();
-        Language currentLanguage = options.getLanguage();
-
-        if (currentLanguage != null && (languageIdentifier == null || languageIdentifier.equals(currentLanguage.getIdentifier()))) {
-            // Ensure that we do not rely on the ServiceLoader API. We can also load an arbitrary language via Options
-            options.setLanguageDefaults(currentLanguage);
-            return currentLanguage;
-        }
-
-        Language language = LanguageLoader.getLanguage(languageIdentifier).orElseThrow();
-        options.setLanguage(language);
-        options.setLanguageDefaults(language);
-        logger.info("Loaded language {}", language.getName());
-        return language;
-    }
 }

core/src/main/java/de/jplag/JPlagComparison.java

@@ -1,166 +1,74 @@
 package de.jplag;
 
-import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
-import java.util.Objects;
 
 /**
- * This method represents the whole result of a comparison between two submissions.
+ * This record represents the whole result of a comparison between two submissions.
+ * @param firstSubmission is the first of the two submissions.
+ * @param secondSubmission is the second of the two submissions.
+ * @param matches is the unmodifiable list of all matches between the two submissions.
  */
-public class JPlagComparison { // FIXME TS: contains a lot of code duplication
-
-    private static final int ROUNDING_FACTOR = 10;
-
-    private final Submission firstSubmission;
-    private final Submission secondSubmission;
-
-    private final List<Match> matches;
-
-    public JPlagComparison(Submission firstSubmission, Submission secondSubmission) {
-        this.firstSubmission = firstSubmission;
-        this.secondSubmission = secondSubmission;
-        matches = new ArrayList<>();
-    }
-
-    /**
-     * Add a match to the comparison (token indices and number of tokens), if it does not overlap with the existing matches.
-     * @see Match#Match(int, int, int)
-     */
-    /* package-private */ final void addMatch(int startOfFirst, int startOfSecond, int length) {
-        for (Match match : matches) {
-            if (match.overlap(startOfFirst, startOfSecond, length)) {
-                return;
-            }
-        }
-        matches.add(new Match(startOfFirst, startOfSecond, length));
-    }
-
-    @Override
-    public boolean equals(Object other) {
-        if (other == this) {
-            return true;
-        }
-        if (!(other instanceof JPlagComparison otherComparison)) {
-            return false;
-        }
-        return firstSubmission.equals(otherComparison.getFirstSubmission()) && secondSubmission.equals(otherComparison.getSecondSubmission())
-                && matches.equals(otherComparison.matches);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(firstSubmission, secondSubmission, matches);
-    }
-
-    /**
-     * @return the base code matches of the first submission.
-     */
-    public JPlagComparison getFirstBaseCodeMatches() {
-        return firstSubmission.getBaseCodeComparison();
-    }
-
-    /**
-     * @return the first of the two submissions.
-     */
-    public Submission getFirstSubmission() {
-        return firstSubmission;
-    }
-
+public record JPlagComparison(Submission firstSubmission, Submission secondSubmission, List<Match> matches) {
     /**
-     * @return all matches between the two submissions.
+     * Initializes a new comparison.
+     * @param firstSubmission is the first of the two submissions.
+     * @param secondSubmission is the second of the two submissions.
+     * @param matches is the list of all matches between the two submissions.
      */
-    public List<Match> getMatches() {
-        return matches;
+    public JPlagComparison(Submission firstSubmission, Submission secondSubmission, List<Match> matches) {
+        this.firstSubmission = firstSubmission;
+        this.secondSubmission = secondSubmission;
+        this.matches = Collections.unmodifiableList(matches);
     }
 
     /**
      * Get the total number of matched tokens for this comparison.
      */
     public final int getNumberOfMatchedTokens() {
-        int numberOfMatchedTokens = 0;
-
-        for (Match match : matches) {
-            numberOfMatchedTokens += match.length();
-        }
-
-        return numberOfMatchedTokens;
-    }
-
-    /**
-     * @return the base code matches of the second submissions.
-     */
-    public JPlagComparison getSecondBaseCodeMatches() {
-        return secondSubmission.getBaseCodeComparison();
-    }
-
-    /**
-     * @return the second of the two submissions.
-     */
-    public Submission getSecondSubmission() {
-        return secondSubmission;
+        return matches.stream().mapToInt(Match::length).sum();
     }
 
     /**
      * @return Maximum similarity in percent of both submissions.
      */
-    public final float maximalSimilarity() {
+    public final double maximalSimilarity() {
         return Math.max(similarityOfFirst(), similarityOfSecond());
     }
 
     /**
      * @return Minimum similarity in percent of both submissions.
      */
-    public final float minimalSimilarity() {
+    public final double minimalSimilarity() {
         return Math.min(similarityOfFirst(), similarityOfSecond());
     }
 
     /**
      * @return Similarity in percent (what percentage of tokens across both submissions are matched).
      */
-    public final float similarity() {
+    public final double similarity() {
         boolean subtractBaseCode = firstSubmission.hasBaseCodeMatches() && secondSubmission.hasBaseCodeMatches();
-        float sa = firstSubmission.getSimilarityDivisor(subtractBaseCode);
-        float sb = secondSubmission.getSimilarityDivisor(subtractBaseCode);
-        return (200 * getNumberOfMatchedTokens()) / (sa + sb);
+        int divisorA = firstSubmission.getSimilarityDivisor(subtractBaseCode);
+        int divisorB = secondSubmission.getSimilarityDivisor(subtractBaseCode);
+        return 2 * similarity(divisorA + divisorB);
     }
 
     /**
      * @return Similarity in percent for the first submission (what percent of the first submission is similar to the
      * second).
      */
-    public final float similarityOfFirst() {
+    public final double similarityOfFirst() {
         int divisor = firstSubmission.getSimilarityDivisor(true);
-        return (divisor == 0 ? 0f : (getNumberOfMatchedTokens() * 100 / (float) divisor));
+        return similarity(divisor);
     }
 
     /**
      * @return Similarity in percent for the second submission (what percent of the second submission is similar to the
      * first).
      */
-    public final float similarityOfSecond() {
+    public final double similarityOfSecond() {
         int divisor = secondSubmission.getSimilarityDivisor(true);
-        return (divisor == 0 ? 0f : (getNumberOfMatchedTokens() * 100 / (float) divisor));
-    }
-
-    /**
-     * @return Similarity in percent rounded down to the nearest tenth.
-     */
-    public final float roundedSimilarity() {
-        return ((int) (similarity() * ROUNDING_FACTOR)) / (float) ROUNDING_FACTOR;
-    }
-
-    /**
-     * @return Similarity of the first submission to the basecode in percent rounded down to the nearest tenth.
-     */
-    public final float basecodeSimilarityOfFirst() {
-        return ((int) (firstBasecodeSimilarity() * ROUNDING_FACTOR)) / (float) ROUNDING_FACTOR;
-    }
-
-    /**
-     * @return Similarity of the second submission to the basecode in percent rounded down to the nearest tenth.
-     */
-    public final float basecodeSimilarityOfSecond() {
-        return ((int) (secondBasecodeSimilarity() * ROUNDING_FACTOR)) / (float) ROUNDING_FACTOR;
+        return similarity(divisor);
     }
 
     @Override
@@ -168,15 +76,7 @@ public class JPlagComparison { // FIXME TS: contains a lot of code duplication
         return firstSubmission.getName() + " <-> " + secondSubmission.getName();
     }
 
-    private float firstBasecodeSimilarity() {
-        float sa = firstSubmission.getSimilarityDivisor(false);
-        JPlagComparison firstBaseCodeMatches = firstSubmission.getBaseCodeComparison();
-        return firstBaseCodeMatches.getNumberOfMatchedTokens() * 100 / sa;
-    }
-
-    private float secondBasecodeSimilarity() {
-        float sb = secondSubmission.getSimilarityDivisor(false);
-        JPlagComparison secondBaseCodeMatches = secondSubmission.getBaseCodeComparison();
-        return secondBaseCodeMatches.getNumberOfMatchedTokens() * 100 / sb;
+    private double similarity(int divisor) {
+        return (divisor == 0 ? 0.0 : (getNumberOfMatchedTokens() * 100 / (double) divisor));
     }
 }

core/src/main/java/de/jplag/JPlagResult.java

@@ -1,7 +1,7 @@
 package de.jplag;
 
 import java.util.List;
-import java.util.function.Function;
+import java.util.function.ToDoubleFunction;
 
 import de.jplag.clustering.ClusteringResult;
 import de.jplag.options.JPlagOptions;
@@ -31,7 +31,7 @@ public class JPlagResult {
         this.durationInMillis = durationInMillis;
         this.options = options;
         similarityDistribution = calculateSimilarityDistribution(comparisons);
-        comparisons.sort((first, second) -> Float.compare(second.similarity(), first.similarity())); // Sort by percentage (descending).
+        comparisons.sort((first, second) -> Double.compare(second.similarity(), first.similarity())); // Sort by percentage (descending).
     }
 
     /**
@@ -61,7 +61,7 @@ public class JPlagResult {
      * @return a list of comparisons sorted descending by percentage.
      */
     public List<JPlagComparison> getComparisons(int numberOfComparisons) {
-        if (numberOfComparisons == -1) {
+        if (numberOfComparisons == JPlagOptions.SHOW_ALL_COMPARISONS) {
             return comparisons;
         }
         return comparisons.subList(0, Math.min(numberOfComparisons, comparisons.size()));
@@ -124,7 +124,7 @@ public class JPlagResult {
     @Override
     public String toString() {
         return String.format("JPlagResult { comparisons: %d, duration: %d ms, language: %s, submissions: %d }", getAllComparisons().size(),
-                getDuration(), getOptions().getLanguage().getName(), submissions.numberOfSubmissions());
+                getDuration(), getOptions().language().getName(), submissions.numberOfSubmissions());
     }
 
     /**
@@ -134,11 +134,11 @@ public class JPlagResult {
         return calculateDistributionFor(comparisons, JPlagComparison::similarity);
     }
 
-    private int[] calculateDistributionFor(List<JPlagComparison> comparisons, Function<JPlagComparison, Float> similarityExtractor) {
+    private int[] calculateDistributionFor(List<JPlagComparison> comparisons, ToDoubleFunction<JPlagComparison> similarityExtractor) {
         int[] similarityDistribution = new int[SIMILARITY_DISTRIBUTION_SIZE];
         int similarityDistributionBucketSize = 100 / SIMILARITY_DISTRIBUTION_SIZE;
         for (JPlagComparison comparison : comparisons) {
-            float similarity = similarityExtractor.apply(comparison); // extract similarity in percent: 0f <= similarity <= 100f
+            double similarity = similarityExtractor.applyAsDouble(comparison); // extract similarity in percent: 0f <= similarity <= 100f
             int index = (int) (similarity / similarityDistributionBucketSize); // divide similarity by bucket size to find index of correct bucket.
             index = Math.min(index, SIMILARITY_DISTRIBUTION_SIZE - 1);// index is out of bounds when similarity is 100%. decrease by one to count
                                                                       // towards the highest value bucket

core/src/main/java/de/jplag/Match.java

@@ -11,21 +11,21 @@ public record Match(int startOfFirst, int startOfSecond, int length) {
      * Checks if two matches overlap.
      * @return true if they do.
      */
-    public boolean overlap(int otherStartOfFirst, int otherStartOfSecond, int otherLength) {
-        if (startOfFirst < otherStartOfFirst) {
-            if ((otherStartOfFirst - startOfFirst) < length) {
+    public boolean overlaps(Match other) {
+        if (startOfFirst < other.startOfFirst) {
+            if ((other.startOfFirst - startOfFirst) < length) {
                 return true;
             }
         } else {
-            if ((startOfFirst - otherStartOfFirst) < otherLength) {
+            if ((startOfFirst - other.startOfFirst) < other.length) {
                 return true;
             }
         }
 
-        if (startOfSecond < otherStartOfSecond) {
-            return (otherStartOfSecond - startOfSecond) < length;
+        if (startOfSecond < other.startOfSecond) {
+            return (other.startOfSecond - startOfSecond) < length;
         } else {
-            return (startOfSecond - otherStartOfSecond) < otherLength;
+            return (startOfSecond - other.startOfSecond) < other.length;
         }
     }
 }

core/src/main/java/de/jplag/Submission.java

@@ -5,6 +5,8 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 import java.util.Objects;
 
 import org.slf4j.Logger;
@@ -49,7 +51,7 @@ public class Submission implements Comparable<Submission> {
     /**
      * Parse result, tokens from all files.
      */
-    private TokenList tokenList;
+    private List<Token> tokenList;
 
     /**
      * Base code comparison
@@ -138,7 +140,7 @@ public class Submission implements Comparable<Submission> {
      * @param subtractBaseCode If true subtract basecode matches if possible.
      * @return Similarity divisor for the submission.
      */
-    public int getSimilarityDivisor(boolean subtractBaseCode) {
+    int getSimilarityDivisor(boolean subtractBaseCode) {
         int divisor = getNumberOfTokens() - getFiles().size();
         if (subtractBaseCode && baseCodeComparison != null) {
             divisor -= baseCodeComparison.getNumberOfMatchedTokens();
@@ -147,10 +149,10 @@ public class Submission implements Comparable<Submission> {
     }
 
     /**
-     * @return Parse result of the submission.
+     * @return unmodifiable list of tokens generated by parsing the submission.
      */
-    public TokenList getTokenList() {
-        return tokenList;
+    public List<Token> getTokenList() {
+        return tokenList == null ? null : Collections.unmodifiableList(tokenList);
     }
 
     /**
@@ -174,15 +176,6 @@ public class Submission implements Comparable<Submission> {
         return isNew;
     }
 
-    /**
-     * Resets the base code flag for all tokens of this submission.
-     */
-    public void resetBaseCode() {
-        for (Token token : tokenList.allTokens()) {
-            token.setBasecode(false);
-        }
-    }
-
     /**
      * Sets the base code comparison
      * @param baseCodeComparison is submissions matches with the base code
@@ -195,7 +188,7 @@ public class Submission implements Comparable<Submission> {
      * Sets the tokens that have been parsed from the files this submission consists of.
      * @param tokenList is the list of these tokens.
      */
-    public void setTokenList(TokenList tokenList) {
+    public void setTokenList(List<Token> tokenList) {
         this.tokenList = tokenList;
     }
 

core/src/main/java/de/jplag/SubmissionSet.java

@@ -115,9 +115,9 @@ public class SubmissionSet {
     private void parseBaseCodeSubmission(Submission baseCode) throws BasecodeException {
         long startTime = System.currentTimeMillis();
         logger.info("----- Parsing basecode submission: " + baseCode.getName());
-        if (!baseCode.parse(options.isDebugParser())) {
+        if (!baseCode.parse(options.debugParser())) {
             throw new BasecodeException("Could not successfully parse basecode submission!");
-        } else if (baseCode.getNumberOfTokens() < options.getMinimumTokenMatch()) {
+        } else if (baseCode.getNumberOfTokens() < options.minimumTokenMatch()) {
             throw new BasecodeException("Basecode submission contains fewer tokens than minimum match length allows!");
         }
         logger.info("Basecode submission parsed!");
@@ -144,11 +144,11 @@ public class SubmissionSet {
             logger.trace("------ Parsing submission: " + submission.getName());
             currentSubmissionName = submission.getName();
 
-            if (!(ok = submission.parse(options.isDebugParser()))) {
+            if (!(ok = submission.parse(options.debugParser()))) {
                 errors++;
             }
 
-            if (submission.getTokenList() != null && submission.getNumberOfTokens() < options.getMinimumTokenMatch()) {
+            if (submission.getTokenList() != null && submission.getNumberOfTokens() < options.minimumTokenMatch()) {
                 logger.error("Submission {} contains fewer tokens than minimum match length allows!", currentSubmissionName);
                 submission.setTokenList(null);
                 tooShort++;

core/src/main/java/de/jplag/SubmissionSetBuilder.java

@@ -39,12 +39,11 @@ public class SubmissionSetBuilder {
      * Creates a builder for submission sets.
      * @param language is the language of the submissions.
      * @param options are the configured options.
-     * @param excludedFileNames a list of file names to be excluded
      */
-    public SubmissionSetBuilder(Language language, JPlagOptions options, Set<String> excludedFileNames) {
+    public SubmissionSetBuilder(Language language, JPlagOptions options) {
         this.language = language;
         this.options = options;
-        this.excludedFileNames = excludedFileNames;
+        this.excludedFileNames = options.excludedFiles();
     }
 
     /**
@@ -53,8 +52,8 @@ public class SubmissionSetBuilder {
      * @throws ExitException if the directory cannot be read.
      */
     public SubmissionSet buildSubmissionSet() throws ExitException {
-        Set<File> submissionDirectories = verifyRootDirectories(options.getSubmissionDirectories(), true);
-        Set<File> oldSubmissionDirectories = verifyRootDirectories(options.getOldSubmissionDirectories(), false);
+        Set<File> submissionDirectories = verifyRootDirectories(options.submissionDirectories(), true);
+        Set<File> oldSubmissionDirectories = verifyRootDirectories(options.oldSubmissionDirectories(), false);
         checkForNonOverlappingRootDirectories(submissionDirectories, oldSubmissionDirectories);
 
         // For backward compatibility, don't prefix submission names with their root directory
@@ -134,7 +133,7 @@ public class SubmissionSetBuilder {
             return Optional.empty();
         }
 
-        String baseCodeName = options.getBaseCodeSubmissionName().orElseThrow();
+        String baseCodeName = Optional.ofNullable(options.baseCodeSubmissionName()).orElseThrow();
         Submission baseCode = loadBaseCodeAsPath(baseCodeName);
         if (baseCode == null) {
             int numberOfRootDirectories = submissionDirectories.size() + oldSubmissionDirectories.size();
@@ -275,17 +274,17 @@ public class SubmissionSetBuilder {
      */
     private Submission processSubmission(String submissionName, File submissionFile, boolean isNew) throws ExitException {
 
-        if (submissionFile.isDirectory() && options.getSubdirectoryName() != null) {
+        if (submissionFile.isDirectory() && options.subdirectoryName() != null) {
             // Use subdirectory instead
-            submissionFile = new File(submissionFile, options.getSubdirectoryName());
+            submissionFile = new File(submissionFile, options.subdirectoryName());
 
             if (!submissionFile.exists()) {
                 throw new SubmissionException(
-                        String.format("Submission %s does not contain the given subdirectory '%s'", submissionName, options.getSubdirectoryName()));
+                        String.format("Submission %s does not contain the given subdirectory '%s'", submissionName, options.subdirectoryName()));
             }
 
             if (!submissionFile.isDirectory()) {
-                throw new SubmissionException(String.format("The given subdirectory '%s' is not a directory!", options.getSubdirectoryName()));
+                throw new SubmissionException(String.format("The given subdirectory '%s' is not a directory!", options.subdirectoryName()));
             }
         }
 
@@ -322,13 +321,13 @@ public class SubmissionSetBuilder {
      * @return true if the file suffix matches the language.
      */
     private boolean hasValidSuffix(File file) {
-        String[] validSuffixes = options.getFileSuffixes();
+        List<String> validSuffixes = options.fileSuffixes();
 
         // This is the case if either the language frontends or the CLI did not set the valid suffixes array in options
-        if (validSuffixes == null || validSuffixes.length == 0) {
+        if (validSuffixes == null || validSuffixes.isEmpty()) {
             return true;
         }
-        return Arrays.stream(validSuffixes).anyMatch(suffix -> file.getName().endsWith(suffix));
+        return validSuffixes.stream().anyMatch(suffix -> file.getName().endsWith(suffix));
     }
 
     /**

core/src/main/java/de/jplag/SubsequenceHashLookupTable.java

@@ -0,0 +1,126 @@
+package de.jplag;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A class to generate and store hashes over a fixed length subsequence of a given list of tokens. Hash generation is
+ * optimized to work in O(n).
+ */
+class SubsequenceHashLookupTable {
+    /**
+     * Value combination is chosen such that the maximum possible hash value does not exceed Int.max. Computation formula
+     * for maximum hash value is \sum from (i=0 to MAX_HASH_LENGTH - 1) with (TOKEN_HASH_MODULO - 1) * 2^i
+     */
+    private static final int MAX_HASH_LENGTH = 25;
+    private static final int TOKEN_HASH_MODULO = 64;
+
+    /** Indicator that the subsequence should not be considered for comparison matching */
+    public static final int NO_HASH = -1;
+
+    private final int windowSize;
+    private final List<Token> tokens;
+    private int[] subsequenceHashes;
+    private Map<Integer, List<Integer>> startIndexToSubsequenceHashesMap;
+
+    /**
+     * Generates a new subsequence hash lookup table. Performance is optimized to compute hashes in O(n).
+     * @param windowSize the size of the subsequences.
+     * @param tokens the tokens to hash over.
+     * @param markedTokens the set of marked tokens. Subsequences containing a marked token obtain the {@link #NO_HASH}
+     * value.
+     */
+    SubsequenceHashLookupTable(int windowSize, List<Token> tokens, Set<Token> markedTokens) {
+        windowSize = Math.max(1, windowSize);
+        windowSize = Math.min(MAX_HASH_LENGTH, windowSize);
+        this.windowSize = windowSize;
+        this.tokens = tokens;
+
+        if (tokens.size() < windowSize) {
+            return;
+        }
+
+        subsequenceHashes = new int[tokens.size() - windowSize];
+        startIndexToSubsequenceHashesMap = new HashMap<>(subsequenceHashes.length);
+        computeSubsequenceHashes(markedTokens);
+    }
+
+    /** Returns the size of the subsequences used for hashing */
+    int getWindowSize() {
+        return windowSize;
+    }
+
+    /** Returns the list of tokens for which the hashes were computed */
+    List<Token> getTokens() {
+        return tokens;
+    }
+
+    /**
+     * Returns the hash over the subsequence from startIndex to startIndex+windowSize.
+     * @param startIndex the start index.
+     * @return the hash of the requested subsequence.
+     */
+    int subsequenceHashForStartIndex(int startIndex) {
+        return subsequenceHashes[startIndex];
+    }
+
+    /**
+     * Returns a list of all start indexes of possible subsequences for the given subsequence hash.
+     * @param subsequenceHash the hash value to obtain possibly matching subsequence start indexes for.
+     * @return a list with possible matching start indexes.
+     */
+    List<Integer> startIndexesOfPossiblyMatchingSubsequencesForSubsequenceHash(int subsequenceHash) {
+        if (startIndexToSubsequenceHashesMap.containsKey(subsequenceHash)) {
+            return startIndexToSubsequenceHashesMap.get(subsequenceHash);
+        }
+        return List.of();
+    }
+
+    /**
+     * Creates hashes for all subsequences with windowSize. Code is optimized to perform in O(n) using a windowing approach.
+     * Hashes are computed by \sum from (i=0 to windowSize) with hash(tokens[offset+i]) * 2^(hashLength-1-i)
+     * @param markedTokens contains the marked tokens. Subsequences containing a marked token will receive the NO_HASH
+     * value.
+     */
+    private void computeSubsequenceHashes(Set<Token> markedTokens) {
+        int hash = 0;
+        int hashedLength = 0;
+        int factor = (windowSize != 1 ? (2 << (windowSize - 2)) : 1);
+
+        for (int windowEndIndex = 0; windowEndIndex < tokens.size(); windowEndIndex++) {
+            int windowStartIndex = windowEndIndex - windowSize;
+            if (windowStartIndex >= 0) {
+                if (hashedLength >= windowSize) {
+                    subsequenceHashes[windowStartIndex] = hash;
+                    addToStartIndexesToHashesMap(windowStartIndex, hash);
+                } else {
+                    subsequenceHashes[windowStartIndex] = NO_HASH;
+                }
+                hash -= factor * (hashValueForToken(tokens.get(windowStartIndex)));
+            }
+            hash = (2 * hash) + (hashValueForToken(tokens.get(windowEndIndex)));
+            if (markedTokens.contains(tokens.get(windowEndIndex))) {
+                hashedLength = 0;
+            } else {
+                hashedLength++;
+            }
+        }
+    }
+
+    private int hashValueForToken(Token token) {
+        return token.type % TOKEN_HASH_MODULO;
+    }
+
+    private void addToStartIndexesToHashesMap(int startIndex, int subsequenceHash) {
+        if (startIndexToSubsequenceHashesMap.containsKey(subsequenceHash)) {
+            startIndexToSubsequenceHashesMap.get(subsequenceHash).add(startIndex);
+        } else {
+            List<Integer> startIndexes = new ArrayList<>();
+            startIndexes.add(startIndex);
+            startIndexToSubsequenceHashesMap.put(subsequenceHash, startIndexes);
+        }
+    }
+}

core/src/main/java/de/jplag/clustering/Cluster.java

@@ -11,17 +11,17 @@ import java.util.function.BiFunction;
  */
 public class Cluster<T> {
 
-    private final float communityStrength;
+    private final double communityStrength;
     private final Collection<T> members;
     private ClusteringResult<T> clusteringResult = null;
-    private final float averageSimilarity;
+    private final double averageSimilarity;
 
     /**
      * @param members Members of the cluster.
      * @param communityStrength A metric of how strongly the members of this cluster are connected.
      * @param averageSimilarity The average similarity between all tuple comparisons of the members in this cluster.
      */
-    public Cluster(Collection<T> members, float communityStrength, float averageSimilarity) {
+    public Cluster(Collection<T> members, double communityStrength, double averageSimilarity) {
         this.members = new ArrayList<>(members);
         this.communityStrength = communityStrength;
         this.averageSimilarity = averageSimilarity;
@@ -37,7 +37,7 @@ public class Cluster<T> {
     /**
      * @return average similarity between all tuple comparisons of the members in this cluster.
      */
-    public float getAverageSimilarity() {
+    public double getAverageSimilarity() {
         return averageSimilarity;
     }
 
@@ -45,7 +45,7 @@ public class Cluster<T> {
      * See {@link ClusteringResult#getCommunityStrength}
      * @return community strength of the cluster
      */
-    public float getCommunityStrength() {
+    public double getCommunityStrength() {
         return communityStrength;
     }
 
@@ -61,7 +61,7 @@ public class Cluster<T> {
     /**
      * @return How much each member of this cluster contributes to the {@link ClusteringResult#getCommunityStrength}.
      */
-    public float getCommunityStrengthPerConnection() {
+    public double getCommunityStrengthPerConnection() {
         int size = members.size();
         if (size < 2)
             return 0;
@@ -74,9 +74,9 @@ public class Cluster<T> {
      * non-clusters. This method may only be called on clusters that are part of a ClusteringResult.
      * @return normalized community strength per connection
      */
-    public float getNormalizedCommunityStrengthPerConnection() {
+    public double getNormalizedCommunityStrengthPerConnection() {
         List<Cluster<T>> goodClusters = clusteringResult.getClusters().stream().filter(cluster -> cluster.getCommunityStrength() > 0).toList();
-        float posCommunityStrengthSum = (float) goodClusters.stream().mapToDouble(Cluster::getCommunityStrengthPerConnection).sum();
+        double posCommunityStrengthSum = goodClusters.stream().mapToDouble(Cluster::getCommunityStrengthPerConnection).sum();
 
         int size = clusteringResult.getClusters().size();
         if (size < 2)
@@ -87,7 +87,7 @@ public class Cluster<T> {
     /**
      * How much this cluster is worth during optimization.
      */
-    public double getWorth(BiFunction<T, T, Float> similarity) {
+    public double getWorth(BiFunction<T, T, Double> similarity) {
         double communityStrength = getCommunityStrength();
         if (members.size() > 1) {
             communityStrength /= connections();
@@ -101,12 +101,12 @@ public class Cluster<T> {
      * @param similarity function that supplies the similarity of two cluster members.
      * @return average similarity
      */
-    private float averageSimilarity(BiFunction<T, T, Float> similarity) {
+    private double averageSimilarity(BiFunction<T, T, Double> similarity) {
         List<T> members = new ArrayList<>(this.members);
         if (members.size() < 2) {
             return 1;
         }
-        float similaritySum = 0;
+        double similaritySum = 0;
         for (int i = 0; i < members.size(); i++) {
             for (int j = i + 1; j < members.size(); j++) {
                 similaritySum += similarity.apply(members.get(i), members.get(j));

core/src/main/java/de/jplag/clustering/ClusteringAdapter.java

@@ -3,7 +3,7 @@ package de.jplag.clustering;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.function.Function;
+import java.util.function.ToDoubleFunction;
 
 import org.apache.commons.math3.linear.Array2DRowRealMatrix;
 import org.apache.commons.math3.linear.RealMatrix;
@@ -30,19 +30,19 @@ public class ClusteringAdapter {
      * @param comparisons that should be included in the process of clustering
      * @param metric function that assigns a similarity to each comparison
      */
-    public ClusteringAdapter(Collection<JPlagComparison> comparisons, Function<JPlagComparison, Float> metric) {
+    public ClusteringAdapter(Collection<JPlagComparison> comparisons, ToDoubleFunction<JPlagComparison> metric) {
         mapping = new IntegerMapping<>(comparisons.size());
         for (JPlagComparison comparison : comparisons) {
-            mapping.map(comparison.getFirstSubmission());
-            mapping.map(comparison.getSecondSubmission());
+            mapping.map(comparison.firstSubmission());
+            mapping.map(comparison.secondSubmission());
         }
         int size = mapping.size();
 
         similarityMatrix = new Array2DRowRealMatrix(size, size);
         for (JPlagComparison comparison : comparisons) {
-            int firstIndex = mapping.map(comparison.getFirstSubmission());
-            int secondIndex = mapping.map(comparison.getSecondSubmission());
-            float similarity = metric.apply(comparison);
+            int firstIndex = mapping.map(comparison.firstSubmission());
+            int secondIndex = mapping.map(comparison.secondSubmission());
+            double similarity = metric.applyAsDouble(comparison);
             similarityMatrix.setEntry(firstIndex, secondIndex, similarity);
             similarityMatrix.setEntry(secondIndex, firstIndex, similarity);
         }

core/src/main/java/de/jplag/clustering/ClusteringFactory.java

@@ -24,18 +24,18 @@ public class ClusteringFactory {
     private static final Logger logger = LoggerFactory.getLogger(ClusteringFactory.class);
 
     public static List<ClusteringResult<Submission>> getClusterings(Collection<JPlagComparison> comparisons, ClusteringOptions options) {
-        if (!options.isEnabled()) {
+        if (!options.enabled()) {
             logger.warn(CLUSTERING_DISABLED);
             return Collections.emptyList();
         } else {
-            logger.info(CLUSTERING_PARAMETERS, options.getAlgorithm(), options.getPreprocessor());
+            logger.info(CLUSTERING_PARAMETERS, options.algorithm(), options.preprocessor());
         }
 
         // init algorithm
-        GenericClusteringAlgorithm clusteringAlgorithm = options.getAlgorithm().create(options);
+        GenericClusteringAlgorithm clusteringAlgorithm = options.algorithm().create(options);
 
         // init preprocessor
-        Optional<ClusteringPreprocessor> preprocessor = options.getPreprocessor().constructPreprocessor(options);
+        Optional<ClusteringPreprocessor> preprocessor = options.preprocessor().constructPreprocessor(options);
 
         if (preprocessor.isPresent()) {
             // Package preprocessor into a clustering algorithm
@@ -43,7 +43,7 @@ public class ClusteringFactory {
         }
 
         // init adapter
-        ClusteringAdapter adapter = new ClusteringAdapter(comparisons, options.getSimilarityMetric());
+        ClusteringAdapter adapter = new ClusteringAdapter(comparisons, options.similarityMetric());
 
         // run clustering
         ClusteringResult<Submission> result = adapter.doClustering(clusteringAlgorithm);
@@ -62,7 +62,7 @@ public class ClusteringFactory {
 
     private static void logClusters(ClusteringResult<Submission> result) {
         var clusters = new ArrayList<>(result.getClusters());
-        Collections.sort(clusters, (first, second) -> Float.compare(second.getCommunityStrength(), first.getCommunityStrength()));
+        clusters.sort((first, second) -> Double.compare(second.getCommunityStrength(), first.getCommunityStrength()));
         logger.info(CLUSTERING_RESULT, clusters.size());
         clusters.forEach(it -> logger.info(CLUSTER_PATTERN, it.getCommunityStrength(), it.getAverageSimilarity(), it.getMembers()));
     }

core/src/main/java/de/jplag/clustering/ClusteringOptions.java

@@ -1,253 +1,143 @@
 
 package de.jplag.clustering;
 
+import java.util.Objects;
+
 import de.jplag.clustering.algorithm.InterClusterSimilarity;
 import de.jplag.options.SimilarityMetric;
 
 /**
  * Collection of all possible parameters that describe how a clustering should be performed.
+ * @param similarityMetric The similarity metric is used for clustering
+ * @param spectralKernelBandwidth The kernel bandwidth for the matern kernel used in the gaussian process for the
+ * automatic search for the number of clusters in spectral clustering. Affects the runtime and results of the spectral
+ * clustering.
+ * @param spectralGaussianProcessVariance This is the assumed level of noise in the evaluation results of a spectral
+ * clustering. Acts as normalization parameter for the Gaussian Process. The default setting works well with similarity
+ * scores in the range between zero and one. Affects the results of the spectral clustering.
+ * @param spectralMinRuns The minimal number of times the kMeans algorithm is run for the spectral clustering. These
+ * runs will use predefined numbers of clusters and will not use the bayesian optimization to determine the number of
+ * clusters.
+ * @param spectralMaxRuns The maximal number of times the kMeans algorithm is run during the bayesian optimization for
+ * spectral clustering. The bayesian optimization may be stopped before, when no more maxima of the acquisition-function
+ * are found.
+ * @param spectralMaxKMeansIterationPerRun Maximum number of iterations of the kMeans clustering per run during spectral
+ * clustering.
+ * @param agglomerativeThreshold Agglomerative clustering will merge clusters that have a similarity higher than this
+ * threshold.
+ * @param preprocessor Preprocessing for the similarity values before clustering. Preprocessing is mandatory for
+ * spectral clustering and optional for agglomerative clustering.
+ * @param enabled whether clustering should be performed
+ * @param algorithm the clustering algorithm to use
+ * @param agglomerativeInterClusterSimilarity Similarity measure between clusters in agglomerative clustering
+ * @param preprocessorThreshold up to which similarity the threshold-preprocessor zeroes out the similarities
+ * @param preprocessorPercentile up to which percentile of similarities the percentile-preprocessor zeroes out the
+ * similarities
  */
-public class ClusteringOptions {
-
-    public static final ClusteringOptions DEFAULTS = new Builder().build();
-
-    private final SimilarityMetric similarityMetric;
-    private final float spectralKernelBandwidth;
-    private final float spectralGaussianProcessVariance;
-    private final int spectralMinRuns;
-    private final int spectralMaxRuns;
-    private final int spectralMaxKMeansIterationPerRun;
-    private final float agglomerativeThreshold;
-    private final Preprocessing preprocessor;
-    private final boolean enabled;
-    private final ClusteringAlgorithm algorithm;
-    private final InterClusterSimilarity agglomerativeInterClusterSimilarity;
-    private final float preprocessorThreshold;
-    private final float preprocessorPercentile;
-
-    /**
-     * @return The similarity metric is used for clustering
-     */
-    public SimilarityMetric getSimilarityMetric() {
-        return similarityMetric;
+public record ClusteringOptions(SimilarityMetric similarityMetric, double spectralKernelBandwidth, double spectralGaussianProcessVariance,
+        int spectralMinRuns, int spectralMaxRuns, int spectralMaxKMeansIterationPerRun, double agglomerativeThreshold, Preprocessing preprocessor,
+        boolean enabled, ClusteringAlgorithm algorithm, InterClusterSimilarity agglomerativeInterClusterSimilarity, double preprocessorThreshold,
+        double preprocessorPercentile) {
+
+    public ClusteringOptions(SimilarityMetric similarityMetric, double spectralKernelBandwidth, double spectralGaussianProcessVariance,
+            int spectralMinRuns, int spectralMaxRuns, int spectralMaxKMeansIterationPerRun, double agglomerativeThreshold, Preprocessing preprocessor,
+            boolean enabled, ClusteringAlgorithm algorithm, InterClusterSimilarity agglomerativeInterClusterSimilarity, double preprocessorThreshold,
+            double preprocessorPercentile) {
+        this.similarityMetric = Objects.requireNonNull(similarityMetric);
+        this.spectralKernelBandwidth = spectralKernelBandwidth;
+        this.spectralGaussianProcessVariance = spectralGaussianProcessVariance;
+        this.spectralMinRuns = spectralMinRuns;
+        this.spectralMaxRuns = spectralMaxRuns;
+        this.spectralMaxKMeansIterationPerRun = spectralMaxKMeansIterationPerRun;
+        this.agglomerativeThreshold = agglomerativeThreshold;
+        this.preprocessor = Objects.requireNonNull(preprocessor);
+        this.enabled = enabled;
+        this.algorithm = Objects.requireNonNull(algorithm);
+        this.agglomerativeInterClusterSimilarity = Objects.requireNonNull(agglomerativeInterClusterSimilarity);
+        this.preprocessorThreshold = preprocessorThreshold;
+        this.preprocessorPercentile = preprocessorPercentile;
     }
 
-    /**
-     * The kernel bandwidth for the matern kernel used in the gaussian process for the automatic search for the number of
-     * clusters in spectral clustering. Affects the runtime and results of the spectral clustering.
-     * @return kernel bandwidth for spectral clustering
-     */
-    public float getSpectralKernelBandwidth() {
-        return spectralKernelBandwidth;
+    public ClusteringOptions() {
+        this(SimilarityMetric.MAX, 20.f, 0.05 * 0.05, 5, 50, 200, 0.2, Preprocessing.CUMULATIVE_DISTRIBUTION_FUNCTION, true,
+                ClusteringAlgorithm.SPECTRAL, InterClusterSimilarity.AVERAGE, 0.2, 0.5);
     }
 
-    /**
-     * This is the assumed level of noise in the evaluation results of a spectral clustering. Acts as normalization
-     * parameter for the Gaussian Process. The default setting works well with similarity scores in the range between zero
-     * and one. Affects the results of the spectral clustering.
-     * @return assumed variance of noise in Gaussian Process during spectral clustering.
-     */
-    public float getSpectralGaussianProcessVariance() {
-        return spectralGaussianProcessVariance;
+    public ClusteringOptions withSimilarityMetric(SimilarityMetric similarityMetric) {
+        return new ClusteringOptions(similarityMetric, spectralKernelBandwidth, spectralGaussianProcessVariance, spectralMinRuns, spectralMaxRuns,
+                spectralMaxKMeansIterationPerRun, agglomerativeThreshold, preprocessor, enabled, algorithm, agglomerativeInterClusterSimilarity,
+                preprocessorThreshold, preprocessorPercentile);
     }
 
-    /**
-     * The minimal number of times the kMeans algorithm is run for the spectral clustering. These runs will use predefined
-     * numbers of clusters and will not use the bayesian optimization to determine the number of clusters.
-     * @return minimal kMeans runs during spectral clustering
-     */
-    public int getSpectralMinRuns() {
-        return spectralMinRuns;
+    public ClusteringOptions withSpectralKernelBandwidth(double spectralKernelBandwidth) {
+        return new ClusteringOptions(similarityMetric, spectralKernelBandwidth, spectralGaussianProcessVariance, spectralMinRuns, spectralMaxRuns,
+                spectralMaxKMeansIterationPerRun, agglomerativeThreshold, preprocessor, enabled, algorithm, agglomerativeInterClusterSimilarity,
+                preprocessorThreshold, preprocessorPercentile);
     }
 
-    /**
-     * The maximal number of times the kMeans algorithm is run during the bayesian optimization for spectral clustering. The
-     * bayesian optimization may be stopped before, when no more maxima of the acquisition-function are found.
-     * @return maximal kMeans runs during spectral clustering
-     */
-    public int getSpectralMaxRuns() {
-        return spectralMaxRuns;
+    public ClusteringOptions withSpectralGaussianProcessVariance(double spectralGaussianProcessVariance) {
+        return new ClusteringOptions(similarityMetric, spectralKernelBandwidth, spectralGaussianProcessVariance, spectralMinRuns, spectralMaxRuns,
+                spectralMaxKMeansIterationPerRun, agglomerativeThreshold, preprocessor, enabled, algorithm, agglomerativeInterClusterSimilarity,
+                preprocessorThreshold, preprocessorPercentile);
     }
 
-    /**
-     * Maximum number of iterations of the kMeans clustering per run during spectral clustering.
-     * @return maximal kMeans iterations
-     */
-    public int getSpectralMaxKMeansIterationPerRun() {
-        return spectralMaxKMeansIterationPerRun;
+    public ClusteringOptions withSpectralMinRuns(int spectralMinRuns) {
+        return new ClusteringOptions(similarityMetric, spectralKernelBandwidth, spectralGaussianProcessVariance, spectralMinRuns, spectralMaxRuns,
+                spectralMaxKMeansIterationPerRun, agglomerativeThreshold, preprocessor, enabled, algorithm, agglomerativeInterClusterSimilarity,
+                preprocessorThreshold, preprocessorPercentile);
     }
 
-    /**
-     * Agglomerative clustering will merge clusters that have a similarity higher than this threshold.
-     * @return merging threshold for agglomerative clustering
-     */
-    public float getAgglomerativeThreshold() {
-        return agglomerativeThreshold;
+    public ClusteringOptions withSpectralMaxRuns(int spectralMaxRuns) {
+        return new ClusteringOptions(similarityMetric, spectralKernelBandwidth, spectralGaussianProcessVariance, spectralMinRuns, spectralMaxRuns,
+                spectralMaxKMeansIterationPerRun, agglomerativeThreshold, preprocessor, enabled, algorithm, agglomerativeInterClusterSimilarity,
+                preprocessorThreshold, preprocessorPercentile);
     }
 
-    /**
-     * Preprocessing for the similarity values before clustering. Preprocessing is mandatory for spectral clustering and
-     * optional for agglomerative clustering.
-     * @return preprocessor
-     */
-    public Preprocessing getPreprocessor() {
-        return preprocessor;
+    public ClusteringOptions withSpectralMaxKMeansIterationPerRun(int spectralMaxKMeansIterationPerRun) {
+        return new ClusteringOptions(similarityMetric, spectralKernelBandwidth, spectralGaussianProcessVariance, spectralMinRuns, spectralMaxRuns,
+                spectralMaxKMeansIterationPerRun, agglomerativeThreshold, preprocessor, enabled, algorithm, agglomerativeInterClusterSimilarity,
+                preprocessorThreshold, preprocessorPercentile);
     }
 
-    /**
-     * @return whether clustering should be performed
-     */
-    public boolean isEnabled() {
-        return enabled;
+    public ClusteringOptions withAgglomerativeThreshold(double agglomerativeThreshold) {
+        return new ClusteringOptions(similarityMetric, spectralKernelBandwidth, spectralGaussianProcessVariance, spectralMinRuns, spectralMaxRuns,
+                spectralMaxKMeansIterationPerRun, agglomerativeThreshold, preprocessor, enabled, algorithm, agglomerativeInterClusterSimilarity,
+                preprocessorThreshold, preprocessorPercentile);
     }
 
-    /**
-     * @return the clustering algorithm to use
-     */
-    public ClusteringAlgorithm getAlgorithm() {
-        return algorithm;
+    public ClusteringOptions withPreprocessor(Preprocessing preprocessor) {
+        return new ClusteringOptions(similarityMetric, spectralKernelBandwidth, spectralGaussianProcessVariance, spectralMinRuns, spectralMaxRuns,
+                spectralMaxKMeansIterationPerRun, agglomerativeThreshold, preprocessor, enabled, algorithm, agglomerativeInterClusterSimilarity,
+                preprocessorThreshold, preprocessorPercentile);
     }
 
-    /**
-     * Similarity measure between clusters in agglomerative clustering.
-     * @return similarity measure
-     */
-    public InterClusterSimilarity getAgglomerativeInterClusterSimilarity() {
-        return agglomerativeInterClusterSimilarity;
+    public ClusteringOptions withEnabled(boolean enabled) {
+        return new ClusteringOptions(similarityMetric, spectralKernelBandwidth, spectralGaussianProcessVariance, spectralMinRuns, spectralMaxRuns,
+                spectralMaxKMeansIterationPerRun, agglomerativeThreshold, preprocessor, enabled, algorithm, agglomerativeInterClusterSimilarity,
+                preprocessorThreshold, preprocessorPercentile);
     }
 
-    /**
-     * @return up to which similarity the threshold-preprocessor zeroes out the similarities
-     */
-    public float getPreprocessorThreshold() {
-        return preprocessorThreshold;
+    public ClusteringOptions withAlgorithm(ClusteringAlgorithm algorithm) {
+        return new ClusteringOptions(similarityMetric, spectralKernelBandwidth, spectralGaussianProcessVariance, spectralMinRuns, spectralMaxRuns,
+                spectralMaxKMeansIterationPerRun, agglomerativeThreshold, preprocessor, enabled, algorithm, agglomerativeInterClusterSimilarity,
+                preprocessorThreshold, preprocessorPercentile);
     }
 
-    /**
-     * @return up to which percentile of similarities the percentile-preprocessor zeroes out the similarities
-     */
-    public float getPreprocessorPercentile() {
-        return preprocessorPercentile;
+    public ClusteringOptions withAgglomerativeInterClusterSimilarity(InterClusterSimilarity agglomerativeInterClusterSimilarity) {
+        return new ClusteringOptions(similarityMetric, spectralKernelBandwidth, spectralGaussianProcessVariance, spectralMinRuns, spectralMaxRuns,
+                spectralMaxKMeansIterationPerRun, agglomerativeThreshold, preprocessor, enabled, algorithm, agglomerativeInterClusterSimilarity,
+                preprocessorThreshold, preprocessorPercentile);
     }
 
-    public static class Builder {
-
-        private SimilarityMetric similarityMetric;
-        private float spectralKernelBandwidth;
-        private float spectralGaussianProcessVariance;
-        private int spectralMinRuns;
-        private int spectralMaxRuns;
-        private int spectralMaxKMeansIterationPerRun;
-        private float agglomerativeThreshold;
-        private Preprocessing preprocessor;
-        private boolean enabled;
-        private ClusteringAlgorithm algorithm;
-        private InterClusterSimilarity agglomerativeInterClusterSimilarity;
-        private float preprocessorThreshold;
-        private float preprocessorPercentile;
-
-        public Builder() {
-            // Setting the defaults here
-            similarityMetric(SimilarityMetric.MAX);
-            spectralKernelBandwidth(20.f);
-            spectralGaussianProcessVariance(0.05f * 0.05f);
-            spectralMinRuns(5);
-            spectralMaxRuns(50);
-            spectralMaxKMeansIterationPerRun(200);
-            agglomerativeThreshold(0.2f);
-            preprocessor(Preprocessing.CUMULATIVE_DISTRIBUTION_FUNCTION);
-            enabled(true);
-            algorithm(ClusteringAlgorithm.SPECTRAL);
-            agglomerativeInterClusterSimilarity(InterClusterSimilarity.AVERAGE);
-            preprocessorThreshold(0.2f);
-            preprocessorPercentile(0.5f);
-        }
-
-        public Builder similarityMetric(SimilarityMetric similarityMetric) {
-            this.similarityMetric = similarityMetric;
-            return Builder.this;
-        }
-
-        public Builder spectralKernelBandwidth(float spectralKernelBandwidth) {
-            this.spectralKernelBandwidth = spectralKernelBandwidth;
-            return Builder.this;
-        }
-
-        public Builder spectralGaussianProcessVariance(float spectralGPVariance) {
-            this.spectralGaussianProcessVariance = spectralGPVariance;
-            return Builder.this;
-        }
-
-        public Builder spectralMinRuns(int spectralMinRuns) {
-            this.spectralMinRuns = spectralMinRuns;
-            return Builder.this;
-        }
-
-        public Builder spectralMaxRuns(int spectralMaxRuns) {
-            this.spectralMaxRuns = spectralMaxRuns;
-            return Builder.this;
-        }
-
-        public Builder spectralMaxKMeansIterationPerRun(int spectralMaxKMeansIterationPerRun) {
-            this.spectralMaxKMeansIterationPerRun = spectralMaxKMeansIterationPerRun;
-            return Builder.this;
-        }
-
-        public Builder agglomerativeThreshold(float agglomerativeThreshold) {
-            this.agglomerativeThreshold = agglomerativeThreshold;
-            return Builder.this;
-        }
-
-        public Builder preprocessor(Preprocessing preprocessor) {
-            this.preprocessor = preprocessor;
-            return Builder.this;
-        }
-
-        public Builder enabled(boolean enabled) {
-            this.enabled = enabled;
-            return Builder.this;
-        }
-
-        public Builder algorithm(ClusteringAlgorithm algorithm) {
-            this.algorithm = algorithm;
-            return Builder.this;
-        }
-
-        public Builder agglomerativeInterClusterSimilarity(InterClusterSimilarity agglomerativeInterClusterSimilarity) {
-            this.agglomerativeInterClusterSimilarity = agglomerativeInterClusterSimilarity;
-            return Builder.this;
-        }
-
-        public Builder preprocessorThreshold(float preprocessorThreshold) {
-            this.preprocessorThreshold = preprocessorThreshold;
-            return Builder.this;
-        }
-
-        public Builder preprocessorPercentile(float preprocessorPercentile) {
-            this.preprocessorPercentile = preprocessorPercentile;
-            return Builder.this;
-        }
-
-        public ClusteringOptions build() {
-
-            return new ClusteringOptions(this);
-        }
+    public ClusteringOptions withPreprocessorThreshold(double preprocessorThreshold) {
+        return new ClusteringOptions(similarityMetric, spectralKernelBandwidth, spectralGaussianProcessVariance, spectralMinRuns, spectralMaxRuns,
+                spectralMaxKMeansIterationPerRun, agglomerativeThreshold, preprocessor, enabled, algorithm, agglomerativeInterClusterSimilarity,
+                preprocessorThreshold, preprocessorPercentile);
     }
 
-    private ClusteringOptions(Builder builder) {
-        this.similarityMetric = builder.similarityMetric;
-        this.spectralKernelBandwidth = builder.spectralKernelBandwidth;
-        this.spectralGaussianProcessVariance = builder.spectralGaussianProcessVariance;
-        this.spectralMinRuns = builder.spectralMinRuns;
-        this.spectralMaxRuns = builder.spectralMaxRuns;
-        this.spectralMaxKMeansIterationPerRun = builder.spectralMaxKMeansIterationPerRun;
-        this.agglomerativeThreshold = builder.agglomerativeThreshold;
-        this.preprocessor = builder.preprocessor;
-        this.enabled = builder.enabled;
-        this.algorithm = builder.algorithm;
-        this.agglomerativeInterClusterSimilarity = builder.agglomerativeInterClusterSimilarity;
-        this.preprocessorThreshold = builder.preprocessorThreshold;
-        this.preprocessorPercentile = builder.preprocessorPercentile;
+    public ClusteringOptions withPreprocessorPercentile(double preprocessorPercentile) {
+        return new ClusteringOptions(similarityMetric, spectralKernelBandwidth, spectralGaussianProcessVariance, spectralMinRuns, spectralMaxRuns,
+                spectralMaxKMeansIterationPerRun, agglomerativeThreshold, preprocessor, enabled, algorithm, agglomerativeInterClusterSimilarity,
+                preprocessorThreshold, preprocessorPercentile);
     }
-
 }

core/src/main/java/de/jplag/clustering/ClusteringResult.java

@@ -1,6 +1,12 @@
 package de.jplag.clustering;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.function.BiFunction;
 import java.util.stream.DoubleStream;
 
@@ -14,9 +20,9 @@ import org.apache.commons.math3.linear.RealMatrix;
 public class ClusteringResult<T> {
 
     private final List<Cluster<T>> clusters;
-    private final float communityStrength;
+    private final double communityStrength;
 
-    public ClusteringResult(Collection<Cluster<T>> clusters, float communityStrength) {
+    public ClusteringResult(Collection<Cluster<T>> clusters, double communityStrength) {
         this.clusters = List.copyOf(clusters);
         this.communityStrength = communityStrength;
         for (Cluster<T> cluster : clusters) {
@@ -36,7 +42,7 @@ public class ClusteringResult<T> {
      * 10.1103/PhysRevE.69.026113 It's called modularity in that paper.
      * @return community strength
      */
-    public float getCommunityStrength() {
+    public double getCommunityStrength() {
         return communityStrength;
     }
 
@@ -45,8 +51,8 @@ public class ClusteringResult<T> {
      * @param similarity TODO DF: JAVADOC
      * @return worth
      */
-    public float getWorth(BiFunction<T, T, Float> similarity) {
-        return (float) getClusters().stream().mapToDouble(c -> c.getWorth(similarity)).map(worth -> Double.isFinite(worth) ? worth : 0).average()
+    public double getWorth(BiFunction<T, T, Double> similarity) {
+        return getClusters().stream().mapToDouble(c -> c.getWorth(similarity)).map(worth -> Double.isFinite(worth) ? worth : 0).average()
                 .getAsDouble();
     }
 
@@ -65,7 +71,7 @@ public class ClusteringResult<T> {
             clusterIdx++;
         }
         List<Cluster<Integer>> clusters = new ArrayList<>(clustering.size());
-        float communityStrength = 0;
+        double communityStrength = 0;
         if (!clustering.isEmpty()) {
             RealMatrix percentagesOfSimilaritySums = new Array2DRowRealMatrix(clustering.size(), clustering.size());
             percentagesOfSimilaritySums = percentagesOfSimilaritySums.scalarMultiply(0);
@@ -86,16 +92,16 @@ public class ClusteringResult<T> {
             for (int i = 0; i < clustering.size(); i++) {
                 double outWeightSum = percentagesOfSimilaritySums.getRowVector(i).getL1Norm();
                 double clusterCommunityStrength = percentagesOfSimilaritySums.getEntry(i, i) - outWeightSum * outWeightSum;
-                float averageSimilarity = calculateAverageSimilarityFor(clustering.get(i), similarity);
-                clusters.add(new Cluster<>(clustering.get(i), (float) clusterCommunityStrength, averageSimilarity));
+                double averageSimilarity = calculateAverageSimilarityFor(clustering.get(i), similarity);
+                clusters.add(new Cluster<>(clustering.get(i), clusterCommunityStrength, averageSimilarity));
                 communityStrength += clusterCommunityStrength;
             }
         }
         return new ClusteringResult<>(clusters, communityStrength);
     }
 
-    private static float calculateAverageSimilarityFor(Collection<Integer> cluster, RealMatrix similarityMatrix) {
-        var sumOfSimilarities = 0f;
+    private static double calculateAverageSimilarityFor(Collection<Integer> cluster, RealMatrix similarityMatrix) {
+        double sumOfSimilarities = 0;
         List<Integer> indices = List.copyOf(cluster);
         for (int i = 1; i < cluster.size(); i++) {
             int indexOfSubmission1 = indices.get(i);
@@ -105,16 +111,16 @@ public class ClusteringResult<T> {
             }
         }
         int nMinusOne = cluster.size() - 1;
-        float numberOfComparisons = (nMinusOne * (nMinusOne + 1))
-                / 2f; /*
-                       * Use Gauss sum to calculate number of comparisons in cluster: Given cluster of size n we need Gauss sum of n-1
-                       * comparisons: compare first element of cluster to all other except itself: n-1 comparisons. compare second element to
-                       * all other except itself and first element (as these two were already compared when we processed the first element),
-                       * n-2 comparisons. compare third element to all other but itself and all previously compared: n-3 comparisons and so
-                       * on. when we reach the second to last element we have n-(n-1)=1 comparisons left. when we reach the last element it
-                       * has already been compared to all other. adding up all comparisons we get: (n-1) + (n-2) + (n-3) + ... + (n-(n-1)) =
-                       * Gauss sum of (n-1)
-                       */
+        double numberOfComparisons = (nMinusOne * (nMinusOne + 1)) / 2.0;
+        /*
+         * Use Gauss sum to calculate number of comparisons in cluster: Given cluster of size n we need Gauss sum of n-1
+         * comparisons: compare first element of cluster to all other except itself: n-1 comparisons. compare second element to
+         * all other except itself and first element (as these two were already compared when we processed the first element),
+         * n-2 comparisons. compare third element to all other but itself and all previously compared: n-3 comparisons and so
+         * on. when we reach the second to last element we have n-(n-1)=1 comparisons left. when we reach the last element it
+         * has already been compared to all other. adding up all comparisons we get: (n-1) + (n-2) + (n-3) + ... + (n-(n-1)) =
+         * Gauss sum of (n-1)
+         */
         return sumOfSimilarities / numberOfComparisons;
     }
 

core/src/main/java/de/jplag/clustering/Preprocessing.java

@@ -15,9 +15,9 @@ public enum Preprocessing {
     /** {@link CumulativeDistributionFunctionPreprocessor} */
     CUMULATIVE_DISTRIBUTION_FUNCTION(options -> new CumulativeDistributionFunctionPreprocessor()),
     /** {@link ThresholdPreprocessor} */
-    THRESHOLD(options -> new ThresholdPreprocessor(options.getPreprocessorThreshold())),
+    THRESHOLD(options -> new ThresholdPreprocessor(options.preprocessorThreshold())),
     /** {@link PercentileThresholdProcessor} */
-    PERCENTILE(options -> new PercentileThresholdProcessor(options.getPreprocessorPercentile()));
+    PERCENTILE(options -> new PercentileThresholdProcessor(options.preprocessorPercentile()));
 
     private final Function<ClusteringOptions, ClusteringPreprocessor> constructor;
 

core/src/main/java/de/jplag/clustering/algorithm/AgglomerativeClustering.java

@@ -1,6 +1,11 @@
 package de.jplag.clustering.algorithm;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.apache.commons.math3.linear.RealMatrix;
@@ -44,7 +49,7 @@ public class AgglomerativeClustering implements GenericClusteringAlgorithm {
             Cluster leftCluster = initialClusters.get(leftIndex);
             for (int rightIndex = leftIndex + 1; rightIndex < initialClusters.size(); rightIndex++) {
                 Cluster rightCluster = initialClusters.get(rightIndex);
-                initialSimilarities.add(new ClusterConnection(leftCluster, rightCluster, (float) similarityMatrix.getEntry(leftIndex, rightIndex)));
+                initialSimilarities.add(new ClusterConnection(leftCluster, rightCluster, similarityMatrix.getEntry(leftIndex, rightIndex)));
             }
         }
 
@@ -57,7 +62,7 @@ public class AgglomerativeClustering implements GenericClusteringAlgorithm {
                 // One cluster already part of another cluster
                 continue;
             }
-            if (nearest.similarity < options.getAgglomerativeThreshold()) {
+            if (nearest.similarity < options.agglomerativeThreshold()) {
                 break;
             }
             clusters.remove(nearest.left);
@@ -65,7 +70,7 @@ public class AgglomerativeClustering implements GenericClusteringAlgorithm {
             nearest.left.submissions().addAll(nearest.right.submissions());
             Cluster combined = new Cluster(nearest.left.submissions());
             for (Cluster otherCluster : clusters) {
-                float similarity = options.getAgglomerativeInterClusterSimilarity().clusterSimilarity(combined.submissions, otherCluster.submissions,
+                double similarity = options.agglomerativeInterClusterSimilarity().clusterSimilarity(combined.submissions, otherCluster.submissions,
                         similarityMatrix);
                 similarities.add(new ClusterConnection(combined, otherCluster, similarity));
             }
@@ -75,7 +80,7 @@ public class AgglomerativeClustering implements GenericClusteringAlgorithm {
         return clusters.stream().map(Cluster::submissions).collect(Collectors.toList());
     }
 
-    private record ClusterConnection(Cluster left, Cluster right, float similarity) implements Comparable<ClusterConnection> {
+    private record ClusterConnection(Cluster left, Cluster right, double similarity) implements Comparable<ClusterConnection> {
         @Override
         public int compareTo(ClusterConnection other) {
             return (int) Math.signum(other.similarity - similarity);

core/src/main/java/de/jplag/clustering/algorithm/InterClusterSimilarity.java

@@ -6,14 +6,14 @@ import java.util.function.BinaryOperator;
 import org.apache.commons.math3.linear.RealMatrix;
 
 public enum InterClusterSimilarity {
-    MIN(Float.MAX_VALUE, Math::min),
-    MAX(Float.MIN_VALUE, Math::max),
-    AVERAGE(0, Float::sum);
+    MIN(Double.MAX_VALUE, Math::min),
+    MAX(Double.MIN_VALUE, Math::max),
+    AVERAGE(0, Double::sum);
 
-    private final float neutralElement;
-    private final BinaryOperator<Float> accumulator;
+    private final double neutralElement;
+    private final BinaryOperator<Double> accumulator;
 
-    InterClusterSimilarity(float neutralElement, BinaryOperator<Float> accumulator) {
+    InterClusterSimilarity(double neutralElement, BinaryOperator<Double> accumulator) {
         this.neutralElement = neutralElement;
         this.accumulator = accumulator;
     }
@@ -25,12 +25,12 @@ public enum InterClusterSimilarity {
      * @param similarityMatrix matrix containing similarities
      * @return similarity between the two clusters
      */
-    public float clusterSimilarity(List<Integer> leftCluster, List<Integer> rightCluster, RealMatrix similarityMatrix) {
-        float similarity = this.neutralElement;
+    public double clusterSimilarity(List<Integer> leftCluster, List<Integer> rightCluster, RealMatrix similarityMatrix) {
+        double similarity = this.neutralElement;
 
         for (int leftSubmission : leftCluster) {
             for (int rightSubmission : rightCluster) {
-                float submissionSimilarity = (float) similarityMatrix.getEntry(leftSubmission, rightSubmission);
+                double submissionSimilarity = similarityMatrix.getEntry(leftSubmission, rightSubmission);
                 similarity = this.accumulator.apply(similarity, submissionSimilarity);
             }
         }

core/src/main/java/de/jplag/clustering/algorithm/SpectralClustering.java

@@ -85,9 +85,9 @@ public class SpectralClustering implements GenericClusteringAlgorithm {
         int maxClusters = (int) Math.ceil(dimension / 2.0);
 
         // Find number of clusters using bayesian optimization
-        RealVector lengthScale = new ArrayRealVector(1, options.getSpectralKernelBandwidth());
+        RealVector lengthScale = new ArrayRealVector(1, options.spectralKernelBandwidth());
         BayesianOptimization bo = new BayesianOptimization(new ArrayRealVector(1, minClusters), new ArrayRealVector(1, maxClusters),
-                options.getSpectralMinRuns(), options.getSpectralMaxRuns(), options.getSpectralGaussianProcessVariance(), lengthScale);
+                options.spectralMinRuns(), options.spectralMaxRuns(), options.spectralGaussianProcessVariance(), lengthScale);
         // bo.debug = true;
         BayesianOptimization.OptimizationResult<Collection<Collection<Integer>>> bayesianOptimizationResult = bo.maximize(r -> {
             int clusters = (int) Math.round(r.getEntry(0));
@@ -95,8 +95,7 @@ public class SpectralClustering implements GenericClusteringAlgorithm {
             clusters = Math.min(maxClusters, clusters);
             Collection<Collection<Integer>> clustering = cluster(clusters, dimension, eigenValueIds, eigenDecomposition);
             ClusteringResult<Integer> modularityRes = ClusteringResult.fromIntegerCollections(new ArrayList<>(clustering), similarityMatrix);
-            return new BayesianOptimization.OptimizationResult<>(modularityRes.getWorth((a, b) -> (float) similarityMatrix.getEntry(a, b)),
-                    clustering);
+            return new BayesianOptimization.OptimizationResult<>(modularityRes.getWorth(similarityMatrix::getEntry), clustering);
         });
 
         return bayesianOptimizationResult.getValue();
@@ -116,7 +115,7 @@ public class SpectralClustering implements GenericClusteringAlgorithm {
         List<ClusterableEigenVector> normRows = IntStream.range(0, dimension).filter(i -> concatenatedEigenVectors.getRowVector(i).getNorm() > 0)
                 .mapToObj(row -> new ClusterableEigenVector(row, concatenatedEigenVectors.getRowVector(row).unitVector())).toList();
 
-        Clusterer<ClusterableEigenVector> clusterer = new KMeansPlusPlusClusterer<>(numberOfClusters, options.getSpectralMaxKMeansIterationPerRun());
+        Clusterer<ClusterableEigenVector> clusterer = new KMeansPlusPlusClusterer<>(numberOfClusters, options.spectralMaxKMeansIterationPerRun());
         List<? extends Cluster<ClusterableEigenVector>> clusters = clusterer.cluster(normRows);
         return clusters.stream().map(cluster -> cluster.getPoints().stream().map(eigenVector -> eigenVector.id).collect(Collectors.toList()))
                 .collect(Collectors.toList());

core/src/main/java/de/jplag/clustering/preprocessors/PercentileThresholdProcessor.java

@@ -12,10 +12,10 @@ import de.jplag.clustering.ClusteringPreprocessor;
  */
 public class PercentileThresholdProcessor implements ClusteringPreprocessor {
 
-    private final float percentile;
+    private final double percentile;
     private ThresholdPreprocessor thresholdPreprocessor;
 
-    public PercentileThresholdProcessor(float percentile) {
+    public PercentileThresholdProcessor(double percentile) {
         this.percentile = percentile;
     }
 

core/src/main/java/de/jplag/options/JPlagOptions.java

@@ -1,320 +1,254 @@
 package de.jplag.options;
 
+import static de.jplag.options.Verbosity.LONG;
 import static de.jplag.strategy.ComparisonMode.NORMAL;
 
-import java.io.File;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import de.jplag.JPlag;
 import de.jplag.Language;
 import de.jplag.clustering.ClusteringOptions;
 import de.jplag.strategy.ComparisonMode;
 
-public class JPlagOptions {
+/**
+ * This record defines the options to configure {@link JPlag}.
+ * @param language Language to use when parsing the submissions.
+ * @param minimumTokenMatch Tunes the comparison sensitivity by adjusting the minimum token required to be counted as
+ * matching section. A smaller {@code <n>} increases the sensitivity but might lead to more false-positives.
+ * @param submissionDirectories Directories with new submissions. These must be checked for plagiarism.
+ * @param oldSubmissionDirectories Directories with old submissions to check against.
+ * @param baseCodeSubmissionName Path name of the directory containing the base code.
+ * @param subdirectoryName Example: If the subdirectoryName is 'src', only the code inside submissionDir/src of each
+ * submission will be used for comparison.
+ * @param fileSuffixes List of file suffixes that should be included.
+ * @param exclusionFileName Name of the file that contains the names of files to exclude from comparison.
+ * @param similarityMetric The similarity metric determines how the minimum similarity threshold required for a
+ * comparison (of two submissions) is calculated. This affects which comparisons are stored and thus make it into the
+ * result object.
+ * @param similarityThreshold Percentage value (must be between 0 and 100). Comparisons (of submissions pairs) with a
+ * similarity below this threshold will be ignored. The default value of 0 allows all matches to be stored. This affects
+ * which comparisons are stored and thus make it into the result object. See also {@link #similarityMetric()}.
+ * @param maximumNumberOfComparisons The maximum number of comparisons that will be shown in the generated report. If
+ * set to {@link #SHOW_ALL_COMPARISONS} all comparisons will be shown.
+ * @param clusteringOptions Clustering options
+ * @param comparisonMode Determines which strategy to use for the comparison of submissions.
+ * @param verbosity Level of output verbosity.
+ * @param debugParser If true, submissions that cannot be parsed will be stored in a separate directory.
+ */
+public record JPlagOptions(Language language, Integer minimumTokenMatch, List<String> submissionDirectories, List<String> oldSubmissionDirectories,
+        String baseCodeSubmissionName, String subdirectoryName, List<String> fileSuffixes, String exclusionFileName,
+        SimilarityMetric similarityMetric, double similarityThreshold, int maximumNumberOfComparisons, ClusteringOptions clusteringOptions,
+        ComparisonMode comparisonMode, Verbosity verbosity, boolean debugParser) {
 
-    private static final Logger logger = LoggerFactory.getLogger("JPlag");
     public static final ComparisonMode DEFAULT_COMPARISON_MODE = NORMAL;
-    public static final float DEFAULT_SIMILARITY_THRESHOLD = 0;
+    public static final double DEFAULT_SIMILARITY_THRESHOLD = 0;
     public static final int DEFAULT_SHOWN_COMPARISONS = 30;
-
+    public static final int SHOW_ALL_COMPARISONS = 0;
+    public static final SimilarityMetric DEFAULT_SIMILARITY_METRIC = SimilarityMetric.AVG;
     public static final Charset CHARSET = StandardCharsets.UTF_8;
 
-    /**
-     * The identifier of the language used to parse the submissions.
-     */
-    private final String languageIdentifier;
-
-    /**
-     * Language used to parse the submissions.
-     */
-    private Language language;
-
-    /**
-     * Determines which strategy to use for the comparison of submissions.
-     */
-    private ComparisonMode comparisonMode = DEFAULT_COMPARISON_MODE;
-
-    /**
-     * If true, submissions that cannot be parsed will be stored in a separate directory.
-     */
-    private boolean debugParser = false;
-
-    /**
-     * Array of file suffixes that should be included.
-     */
-    private String[] fileSuffixes;
-
-    /**
-     * Percentage value (must be between 0 and 100). Comparisons (of submissions pairs) with a similarity below this
-     * threshold will be ignored. The default value of 0 allows all matches to be stored. This affects which comparisons are
-     * stored and thus make it into the result object.
-     * @see JPlagOptions#similarityMetric
-     */
-    private float similarityThreshold = DEFAULT_SIMILARITY_THRESHOLD;
-
-    /**
-     * The maximum number of comparisons that will be shown in the generated report. If set to -1 all comparisons will be
-     * shown.
-     */
-    private int maximumNumberOfComparisons = DEFAULT_SHOWN_COMPARISONS;
-
-    /**
-     * The similarity metric determines how the minimum similarity threshold required for a comparison (of two submissions)
-     * is calculated. This affects which comparisons are stored and thus make it into the result object.
-     * @see JPlagOptions#similarityThreshold
-     */
-    private SimilarityMetric similarityMetric = SimilarityMetric.AVG;
-
-    /**
-     * Tunes the comparison sensitivity by adjusting the minimum token required to be counted as matching section. A smaller
-     * <n> increases the sensitivity but might lead to more false-positives.
-     */
-    private Integer minimumTokenMatch;
-
-    /**
-     * Name of the file that contains the names of files to exclude from comparison.
-     */
-    private String exclusionFileName;
-
-    /**
-     * Names of the excluded files.
-     */
-    private Set<String> excludedFiles = Collections.emptySet();
-
-    /**
-     * Directories with new submissions. These must be checked for plagiarism.
-     */
-    private List<String> submissionDirectories;
-
-    /**
-     * Directories with old submissions to check against.
-     */
-    private List<String> oldSubmissionDirectories;
-
-    /**
-     * Path name of the directory containing the base code.
-     * <p>
-     * For backwards compatibility it may also be a directory name inside the root directory. Condition for the latter is
-     * <ul>
-     * <li>Specified path does not exist.</li>
-     * <li>Name has not have a separator character after trimming them from both ends (leaving at least a one-character
-     * name).</li>
-     * <li>A submission with the specified name exists in the root directory.</li>
-     * </ul>
-     * It's an error if a string has been provided but it is neither an existing path nor does it fulfill all the conditions
-     * of the compatibility fallback listed above.
-     * </p>
-     */
-    private String baseCodeSubmissionName = null;
-
-    /**
-     * Example: If the subdirectoryName is 'src', only the code inside submissionDir/src of each submission will be used for
-     * comparison.
-     */
-    private String subdirectoryName;
-
-    /**
-     * Level of output verbosity.
-     */
-    private Verbosity verbosity;
-
-    /**
-     * Clustering options
-     */
-    private ClusteringOptions clusteringOptions = new ClusteringOptions.Builder().build();
-
-    /**
-     * Constructor with required attributes.
-     * @param languageIdentifier the identifier of the language to use. If set to {@code null} you have to use
-     * {@link #setLanguage(Language)} to set the language programmatically.
-     */
-    public JPlagOptions(List<String> submissionDirectories, List<String> oldSubmissionDirectories, String languageIdentifier) {
-        this.submissionDirectories = submissionDirectories;
-        this.oldSubmissionDirectories = oldSubmissionDirectories;
-        this.languageIdentifier = languageIdentifier;
-    }
-
-    public Optional<String> getBaseCodeSubmissionName() {
-        return Optional.ofNullable(baseCodeSubmissionName);
-    }
-
-    public ComparisonMode getComparisonMode() {
-        return comparisonMode;
-    }
+    private static final Logger logger = LoggerFactory.getLogger(JPlag.class);
 
-    public Set<String> getExcludedFiles() {
-        return excludedFiles;
+    public JPlagOptions(Language language, List<String> submissionDirectories, List<String> oldSubmissionDirectories) {
+        this(language, null, submissionDirectories, oldSubmissionDirectories, null, null, null, null, DEFAULT_SIMILARITY_METRIC,
+                DEFAULT_SIMILARITY_THRESHOLD, DEFAULT_SHOWN_COMPARISONS, new ClusteringOptions(), DEFAULT_COMPARISON_MODE, null, false);
     }
 
-    public String getExclusionFileName() {
-        return exclusionFileName;
-    }
-
-    public String[] getFileSuffixes() {
-        return fileSuffixes;
-    }
-
-    public String getLanguageIdentifier() {
-        return languageIdentifier;
-    }
-
-    public Language getLanguage() {
-        return language;
+    public JPlagOptions(Language language, Integer minimumTokenMatch, List<String> submissionDirectories, List<String> oldSubmissionDirectories,
+            String baseCodeSubmissionName, String subdirectoryName, List<String> fileSuffixes, String exclusionFileName,
+            SimilarityMetric similarityMetric, double similarityThreshold, int maximumNumberOfComparisons, ClusteringOptions clusteringOptions,
+            ComparisonMode comparisonMode, Verbosity verbosity, boolean debugParser) {
+        this.language = language;
+        this.comparisonMode = comparisonMode;
+        this.debugParser = debugParser;
+        this.fileSuffixes = fileSuffixes == null || fileSuffixes.isEmpty() ? null : Collections.unmodifiableList(fileSuffixes);
+        this.similarityThreshold = normalizeSimilarityThreshold(similarityThreshold);
+        this.maximumNumberOfComparisons = normalizeMaximumNumberOfComparisons(maximumNumberOfComparisons);
+        this.similarityMetric = similarityMetric;
+        this.minimumTokenMatch = normalizeMinimumTokenMatch(minimumTokenMatch);
+        this.exclusionFileName = exclusionFileName;
+        this.submissionDirectories = submissionDirectories == null ? null : Collections.unmodifiableList(submissionDirectories);
+        this.oldSubmissionDirectories = oldSubmissionDirectories == null ? null : Collections.unmodifiableList(oldSubmissionDirectories);
+        this.baseCodeSubmissionName = (baseCodeSubmissionName == null || baseCodeSubmissionName.isBlank()) ? null : baseCodeSubmissionName;
+        this.subdirectoryName = subdirectoryName;
+        this.verbosity = verbosity;
+        this.clusteringOptions = clusteringOptions;
     }
 
-    public int getMaximumNumberOfComparisons() {
-        return this.maximumNumberOfComparisons;
+    public JPlagOptions withLanguageOption(Language language) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public Integer getMinimumTokenMatch() {
-        return minimumTokenMatch;
+    public JPlagOptions withComparisonMode(ComparisonMode comparisonMode) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public List<String> getSubmissionDirectories() {
-        return submissionDirectories;
+    public JPlagOptions withDebugParser(boolean debugParser) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public List<String> getOldSubmissionDirectories() {
-        return oldSubmissionDirectories;
+    public JPlagOptions withFileSuffixes(List<String> fileSuffixes) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public SimilarityMetric getSimilarityMetric() {
-        return similarityMetric;
+    public JPlagOptions withSimilarityThreshold(double similarityThreshold) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public float getSimilarityThreshold() {
-        return similarityThreshold;
+    public JPlagOptions withMaximumNumberOfComparisons(int maximumNumberOfComparisons) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public String getSubdirectoryName() {
-        return subdirectoryName;
+    public JPlagOptions withSimilarityMetric(SimilarityMetric similarityMetric) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public Verbosity getVerbosity() {
-        return verbosity;
+    public JPlagOptions withMinimumTokenMatch(Integer minimumTokenMatch) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public boolean hasBaseCode() {
-        return this.baseCodeSubmissionName != null;
+    public JPlagOptions withExclusionFileName(String exclusionFileName) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public boolean isDebugParser() {
-        return debugParser;
+    public JPlagOptions withSubmissionDirectories(List<String> submissionDirectories) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public ClusteringOptions getClusteringOptions() {
-        return this.clusteringOptions;
+    public JPlagOptions withOldSubmissionDirectories(List<String> oldSubmissionDirectories) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public void setBaseCodeSubmissionName(String baseCodeSubmissionName) {
-        if (baseCodeSubmissionName == null || baseCodeSubmissionName.isEmpty()) {
-            this.baseCodeSubmissionName = null;
-        } else {
-            this.baseCodeSubmissionName = baseCodeSubmissionName;
-        }
+    public JPlagOptions withBaseCodeSubmissionName(String baseCodeSubmissionName) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public void setComparisonMode(ComparisonMode comparisonMode) {
-        this.comparisonMode = comparisonMode;
+    public JPlagOptions withSubdirectoryName(String subdirectoryName) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public void setDebugParser(boolean debugParser) {
-        this.debugParser = debugParser;
+    public JPlagOptions withVerbosity(Verbosity verbosity) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public void setExcludedFiles(Set<String> excludedFiles) {
-        this.excludedFiles = excludedFiles;
+    public JPlagOptions withClusteringOptions(ClusteringOptions clusteringOptions) {
+        return new JPlagOptions(language, minimumTokenMatch, submissionDirectories, oldSubmissionDirectories, baseCodeSubmissionName,
+                subdirectoryName, fileSuffixes, exclusionFileName, similarityMetric, similarityThreshold, maximumNumberOfComparisons,
+                clusteringOptions, comparisonMode, verbosity, debugParser);
     }
 
-    public void setExclusionFileName(String exclusionFileName) {
-        this.exclusionFileName = exclusionFileName;
+    public boolean hasBaseCode() {
+        return baseCodeSubmissionName != null;
     }
 
-    public void setFileSuffixes(String[] fileSuffixes) {
-        this.fileSuffixes = fileSuffixes;
+    public Set<String> excludedFiles() {
+        return Optional.ofNullable(exclusionFileName()).map(this::readExclusionFile).orElse(Collections.emptySet());
     }
 
-    public void setLanguage(Language language) {
-        this.language = language;
+    @Override
+    public List<String> fileSuffixes() {
+        var language = language();
+        if ((fileSuffixes == null || fileSuffixes.isEmpty()) && language != null)
+            return Arrays.stream(language.suffixes()).toList();
+        return fileSuffixes == null ? null : Collections.unmodifiableList(fileSuffixes);
     }
 
     /**
-     * After the selected language has been initialized, this method is called by JPlag to set default values for options
-     * not set by the user.
-     * @param language - initialized language instance
+     * Path name of the directory containing the base code.<br>
+     * For backwards compatibility it may also be a directory name inside the root directory. Condition for the latter is
+     * <ul>
+     * <li>Specified path does not exist.</li>
+     * <li>Name has not have a separator character after trimming them from both ends (leaving at least a one-character
+     * name).</li>
+     * <li>A submission with the specified name exists in the root directory.</li>
+     * </ul>
+     * It's an error if a string has been provided, but it is neither an existing path nor does it fulfill all the
+     * conditions of the compatibility fallback listed above.
      */
-    public void setLanguageDefaults(Language language) {
-        if (!hasMinimumTokenMatch()) {
-            setMinimumTokenMatch(language.minimumTokenMatch());
-        }
-
-        if (!hasFileSuffixes()) {
-            fileSuffixes = language.suffixes();
-        }
+    @Override
+    public String baseCodeSubmissionName() {
+        return baseCodeSubmissionName;
     }
 
-    public void setMaximumNumberOfComparisons(int maximumNumberOfComparisons) {
-        this.maximumNumberOfComparisons = Math.max(maximumNumberOfComparisons, -1);
+    @Override
+    public Integer minimumTokenMatch() {
+        var language = language();
+        if (minimumTokenMatch == null && language != null)
+            return language.minimumTokenMatch();
+        return minimumTokenMatch;
     }
 
-    public void setMinimumTokenMatch(Integer minimumTokenMatch) {
-        if (minimumTokenMatch != null && minimumTokenMatch < 1) {
-            this.minimumTokenMatch = 1;
-        } else {
-            this.minimumTokenMatch = minimumTokenMatch;
+    private Set<String> readExclusionFile(final String exclusionFileName) {
+        try (BufferedReader reader = new BufferedReader(new FileReader(exclusionFileName, JPlagOptions.CHARSET))) {
+            final var excludedFileNames = reader.lines().collect(Collectors.toSet());
+            if (verbosity() == LONG && logger.isInfoEnabled()) {
+                logger.info("Excluded files:\n{}", String.join("\n", excludedFileNames));
+            }
+            return excludedFileNames;
+        } catch (IOException e) {
+            logger.error("Could not read exclusion file: " + e.getMessage(), e);
+            return Collections.emptySet();
         }
     }
 
-    public void setSubmissionDirectories(List<String> submissionDirectories) {
-        this.submissionDirectories = submissionDirectories;
-    }
-
-    public void setOldSubmissionDirectories(List<String> oldSubmissionDirectories) {
-        this.oldSubmissionDirectories = oldSubmissionDirectories;
-    }
-
-    public void setSimilarityMetric(SimilarityMetric similarityMetric) {
-        this.similarityMetric = similarityMetric;
-    }
-
-    public void setSimilarityThreshold(float similarityThreshold) {
+    private static double normalizeSimilarityThreshold(double similarityThreshold) {
         if (similarityThreshold > 100) {
             logger.warn("Maximum threshold of 100 used instead of {}", similarityThreshold);
-            this.similarityThreshold = 100;
+            return 100;
         } else if (similarityThreshold < 0) {
             logger.warn("Minimum threshold of 0 used instead of {}", similarityThreshold);
-            this.similarityThreshold = 0;
+            return 0;
         } else {
-            this.similarityThreshold = similarityThreshold;
+            return similarityThreshold;
         }
     }
 
-    public void setSubdirectoryName(String subdirectoryName) {
-        // Trim problematic file separators.
-        this.subdirectoryName = (subdirectoryName == null) ? null : subdirectoryName.replace(File.separator, "");
-    }
-
-    public void setVerbosity(Verbosity verbosity) {
-        this.verbosity = verbosity;
-    }
-
-    public void setClusteringOptions(ClusteringOptions clusteringOptions) {
-        this.clusteringOptions = clusteringOptions;
+    private Integer normalizeMaximumNumberOfComparisons(Integer maximumNumberOfComparisons) {
+        return Math.max(maximumNumberOfComparisons, SHOW_ALL_COMPARISONS);
     }
 
-    private boolean hasFileSuffixes() {
-        return fileSuffixes != null && fileSuffixes.length > 0;
+    private Integer normalizeMinimumTokenMatch(Integer minimumTokenMatch) {
+        return (minimumTokenMatch != null && minimumTokenMatch < 1) ? Integer.valueOf(1) : minimumTokenMatch;
     }
-
-    private boolean hasMinimumTokenMatch() {
-        return minimumTokenMatch != null;
-    }
-
 }

core/src/main/java/de/jplag/options/SimilarityMetric.java

@@ -1,27 +1,27 @@
 package de.jplag.options;
 
-import java.util.function.Function;
+import java.util.function.ToDoubleFunction;
 
 import de.jplag.JPlagComparison;
 
-public enum SimilarityMetric implements Function<JPlagComparison, Float> {
+public enum SimilarityMetric implements ToDoubleFunction<JPlagComparison> {
     AVG(JPlagComparison::similarity),
     MIN(JPlagComparison::minimalSimilarity),
     MAX(JPlagComparison::maximalSimilarity),
-    INTERSECTION(it -> (float) it.getNumberOfMatchedTokens());
+    INTERSECTION(it -> (double) it.getNumberOfMatchedTokens());
 
-    private final Function<JPlagComparison, Float> similarityFunction;
+    private final ToDoubleFunction<JPlagComparison> similarityFunction;
 
-    SimilarityMetric(Function<JPlagComparison, Float> determinePercentage) {
+    SimilarityMetric(ToDoubleFunction<JPlagComparison> determinePercentage) {
         this.similarityFunction = determinePercentage;
     }
 
-    public boolean isAboveThreshold(JPlagComparison comparison, float similarityThreshold) {
-        return similarityFunction.apply(comparison) >= similarityThreshold;
+    public boolean isAboveThreshold(JPlagComparison comparison, double similarityThreshold) {
+        return similarityFunction.applyAsDouble(comparison) >= similarityThreshold;
     }
 
     @Override
-    public Float apply(JPlagComparison comparison) {
-        return similarityFunction.apply(comparison);
+    public double applyAsDouble(JPlagComparison comparison) {
+        return similarityFunction.applyAsDouble(comparison);
     }
 }

core/src/main/java/de/jplag/reporting/jsonfactory/ComparisonReportWriter.java

@@ -1,11 +1,14 @@
 package de.jplag.reporting.jsonfactory;
 
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Function;
 
-import de.jplag.*;
+import de.jplag.JPlagComparison;
+import de.jplag.JPlagResult;
+import de.jplag.Submission;
+import de.jplag.Token;
 import de.jplag.reporting.reportobject.model.ComparisonReport;
 import de.jplag.reporting.reportobject.model.Match;
 
@@ -17,7 +20,7 @@ public class ComparisonReportWriter {
 
     private final FileWriter fileWriter;
     private final Function<Submission, String> submissionToIdFunction;
-    private final Map<String, Map<String, String>> submissionIdToComparisonFileName = new HashMap<>();
+    private final Map<String, Map<String, String>> submissionIdToComparisonFileName = new ConcurrentHashMap<>();
 
     public ComparisonReportWriter(Function<Submission, String> submissionToIdFunction, FileWriter fileWriter) {
         this.submissionToIdFunction = submissionToIdFunction;
@@ -35,22 +38,22 @@ public class ComparisonReportWriter {
      * yield the same result.
      */
     public Map<String, Map<String, String>> writeComparisonReports(JPlagResult jPlagResult, String path) {
-        int numberOfComparisons = jPlagResult.getOptions().getMaximumNumberOfComparisons();
+        int numberOfComparisons = jPlagResult.getOptions().maximumNumberOfComparisons();
         List<JPlagComparison> comparisons = jPlagResult.getComparisons(numberOfComparisons);
         writeComparisons(path, comparisons);
         return submissionIdToComparisonFileName;
     }
 
     private void writeComparisons(String path, List<JPlagComparison> comparisons) {
-        for (JPlagComparison comparison : comparisons) {
-            String firstSubmissionId = submissionToIdFunction.apply(comparison.getFirstSubmission());
-            String secondSubmissionId = submissionToIdFunction.apply(comparison.getSecondSubmission());
+        comparisons.parallelStream().forEach(comparison -> {
+            String firstSubmissionId = submissionToIdFunction.apply(comparison.firstSubmission());
+            String secondSubmissionId = submissionToIdFunction.apply(comparison.secondSubmission());
             String fileName = generateComparisonName(firstSubmissionId, secondSubmissionId);
             addToLookUp(firstSubmissionId, secondSubmissionId, fileName);
             var comparisonReport = new ComparisonReport(firstSubmissionId, secondSubmissionId, comparison.similarity(),
                     convertMatchesToReportMatches(comparison));
             fileWriter.saveAsJSON(comparisonReport, path, fileName);
-        }
+        });
     }
 
     private void addToLookUp(String firstSubmissionId, String secondSubmissionId, String fileName) {
@@ -59,7 +62,7 @@ public class ComparisonReportWriter {
     }
 
     private void writeToMap(String id1, String id2, String comparisonFileName) {
-        submissionIdToComparisonFileName.putIfAbsent(id1, new HashMap<>());
+        submissionIdToComparisonFileName.putIfAbsent(id1, new ConcurrentHashMap<>());
         submissionIdToComparisonFileName.get(id1).put(id2, comparisonFileName);
     }
 
@@ -83,16 +86,16 @@ public class ComparisonReportWriter {
     }
 
     private List<Match> convertMatchesToReportMatches(JPlagComparison comparison) {
-        return comparison.getMatches().stream().map(match -> convertMatchToReportMatch(comparison, match)).toList();
+        return comparison.matches().stream().map(match -> convertMatchToReportMatch(comparison, match)).toList();
     }
 
     private Match convertMatchToReportMatch(JPlagComparison comparison, de.jplag.Match match) {
-        TokenList tokensFirst = comparison.getFirstSubmission().getTokenList();
-        TokenList tokensSecond = comparison.getSecondSubmission().getTokenList();
-        Token startOfFirst = tokensFirst.getToken(match.startOfFirst());
-        Token endOfFirst = tokensFirst.getToken(match.startOfFirst() + match.length() - 1);
-        Token startOfSecond = tokensSecond.getToken(match.startOfSecond());
-        Token endOfSecond = tokensSecond.getToken(match.startOfSecond() + match.length() - 1);
+        List<Token> tokensFirst = comparison.firstSubmission().getTokenList();
+        List<Token> tokensSecond = comparison.secondSubmission().getTokenList();
+        Token startOfFirst = tokensFirst.get(match.startOfFirst());
+        Token endOfFirst = tokensFirst.get(match.startOfFirst() + match.length() - 1);
+        Token startOfSecond = tokensSecond.get(match.startOfSecond());
+        Token endOfSecond = tokensSecond.get(match.startOfSecond() + match.length() - 1);
 
         return new Match(startOfFirst.getFile(), startOfSecond.getFile(), startOfFirst.getLine(), endOfFirst.getLine(), startOfSecond.getLine(),
                 endOfSecond.getLine(), match.length());

core/src/main/java/de/jplag/reporting/jsonfactory/DirectoryManager.java

@@ -88,6 +88,8 @@ public class DirectoryManager {
             deleteDirectory(zipName);
             return false;
         }
+        logger.info("Successfully zipped report files: {}", zipName);
+        logger.info("Display the results with the report viewer at https://jplag.github.io/JPlag/");
         return true;
     }
 }

core/src/main/java/de/jplag/reporting/reportobject/ReportObjectFactory.java

@@ -1,6 +1,8 @@
 package de.jplag.reporting.reportobject;
 
-import static de.jplag.reporting.jsonfactory.DirectoryManager.*;
+import static de.jplag.reporting.jsonfactory.DirectoryManager.createDirectory;
+import static de.jplag.reporting.jsonfactory.DirectoryManager.deleteDirectory;
+import static de.jplag.reporting.jsonfactory.DirectoryManager.zipDirectory;
 import static de.jplag.reporting.reportobject.mapper.SubmissionNameToIdMapper.buildSubmissionNameToIdMap;
 
 import java.io.File;
@@ -8,7 +10,11 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
 import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -48,6 +54,7 @@ public class ReportObjectFactory {
     public void createAndSaveReport(JPlagResult result, String path) {
 
         try {
+            logger.info("Start writing report files...");
             createDirectory(path);
             buildSubmissionToIdMap(result);
 
@@ -56,6 +63,7 @@ public class ReportObjectFactory {
             writeComparisons(result, path);
             writeOverview(result, path);
 
+            logger.info("Zipping report files...");
             zipAndDelete(path);
         } catch (IOException e) {
             logger.error("Could not create directory " + path + " for report viewer generation", e);
@@ -78,13 +86,13 @@ public class ReportObjectFactory {
     }
 
     private void copySubmissionFilesToReport(String path, JPlagResult result) {
-        List<JPlagComparison> comparisons = result.getComparisons(result.getOptions().getMaximumNumberOfComparisons());
+        List<JPlagComparison> comparisons = result.getComparisons(result.getOptions().maximumNumberOfComparisons());
         Set<Submission> submissions = getSubmissions(comparisons);
         File submissionsPath = createSubmissionsDirectory(path);
         if (submissionsPath == null) {
             return;
         }
-        Language language = result.getOptions().getLanguage();
+        Language language = result.getOptions().language();
         for (Submission submission : submissions) {
             File directory = createSubmissionDirectory(path, submissionsPath, submission);
             if (directory == null) {
@@ -127,21 +135,21 @@ public class ReportObjectFactory {
     private void writeOverview(JPlagResult result, String path) {
 
         List<String> folders = new ArrayList<>();
-        folders.addAll(result.getOptions().getSubmissionDirectories());
-        folders.addAll(result.getOptions().getOldSubmissionDirectories());
+        folders.addAll(result.getOptions().submissionDirectories());
+        folders.addAll(result.getOptions().oldSubmissionDirectories());
 
-        String baseCodePath = result.getOptions().hasBaseCode() ? result.getOptions().getBaseCodeSubmissionName().orElse("") : "";
+        String baseCodePath = result.getOptions().hasBaseCode() ? result.getOptions().baseCodeSubmissionName() : "";
         ClusteringResultMapper clusteringResultMapper = new ClusteringResultMapper(submissionToIdFunction);
 
         OverviewReport overviewReport = new OverviewReport(folders, // submissionFolderPath
                 baseCodePath, // baseCodeFolderPath
-                result.getOptions().getLanguage().getName(), // language
-                List.of(result.getOptions().getFileSuffixes()), // fileExtensions
+                result.getOptions().language().getName(), // language
+                result.getOptions().fileSuffixes(), // fileExtensions
                 submissionNameToIdMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)), // submissionIds
                 submissionNameToNameToComparisonFileName, // result.getOptions().getMinimumTokenMatch(),
                 List.of(), // failedSubmissionNames
-                result.getOptions().getExcludedFiles(), // excludedFiles
-                result.getOptions().getMinimumTokenMatch(), // matchSensitivity
+                result.getOptions().excludedFiles(), // excludedFiles
+                result.getOptions().minimumTokenMatch(), // matchSensitivity
                 getDate(),// dateOfExecution
                 result.getDuration(), // executionTime
                 getMetrics(result),// metrics
@@ -152,8 +160,8 @@ public class ReportObjectFactory {
     }
 
     private Set<Submission> getSubmissions(List<JPlagComparison> comparisons) {
-        Set<Submission> submissions = comparisons.stream().map(JPlagComparison::getFirstSubmission).collect(Collectors.toSet());
-        Set<Submission> secondSubmissions = comparisons.stream().map(JPlagComparison::getSecondSubmission).collect(Collectors.toSet());
+        Set<Submission> submissions = comparisons.stream().map(JPlagComparison::firstSubmission).collect(Collectors.toSet());
+        Set<Submission> secondSubmissions = comparisons.stream().map(JPlagComparison::secondSubmission).collect(Collectors.toSet());
         submissions.addAll(secondSubmissions);
         return submissions;
     }

core/src/main/java/de/jplag/reporting/reportobject/mapper/MetricMapper.java

@@ -35,7 +35,7 @@ public class MetricMapper {
     }
 
     private List<JPlagComparison> getComparisons(JPlagResult result) {
-        int maxNumberOfComparisons = result.getOptions().getMaximumNumberOfComparisons();
+        int maxNumberOfComparisons = result.getOptions().maximumNumberOfComparisons();
         return result.getComparisons(maxNumberOfComparisons);
     }
 
@@ -43,10 +43,10 @@ public class MetricMapper {
         return Arrays.stream(array).boxed().collect(Collectors.toList());
     }
 
-    private List<TopComparison> getTopComparisons(List<JPlagComparison> comparisons, Function<JPlagComparison, Float> similarityExtractor) {
+    private List<TopComparison> getTopComparisons(List<JPlagComparison> comparisons, Function<JPlagComparison, Double> similarityExtractor) {
         return comparisons.stream().sorted(Comparator.comparing(similarityExtractor).reversed())
-                .map(comparison -> new TopComparison(submissionToIdFunction.apply(comparison.getFirstSubmission()),
-                        submissionToIdFunction.apply(comparison.getSecondSubmission()), similarityExtractor.apply(comparison)))
+                .map(comparison -> new TopComparison(submissionToIdFunction.apply(comparison.firstSubmission()),
+                        submissionToIdFunction.apply(comparison.secondSubmission()), similarityExtractor.apply(comparison)))
                 .toList();
     }
 

core/src/main/java/de/jplag/reporting/reportobject/mapper/SubmissionNameToIdMapper.java

@@ -26,8 +26,8 @@ public class SubmissionNameToIdMapper {
     public static Map<String, String> buildSubmissionNameToIdMap(JPlagResult result) {
         HashMap<String, String> idToName = new HashMap<>();
         getComparisons(result).forEach(comparison -> {
-            idToName.put(comparison.getFirstSubmission().getName(), sanitizeNameOf(comparison.getFirstSubmission()));
-            idToName.put(comparison.getSecondSubmission().getName(), sanitizeNameOf(comparison.getSecondSubmission()));
+            idToName.put(comparison.firstSubmission().getName(), sanitizeNameOf(comparison.firstSubmission()));
+            idToName.put(comparison.secondSubmission().getName(), sanitizeNameOf(comparison.secondSubmission()));
         });
         return idToName;
     }
@@ -37,7 +37,7 @@ public class SubmissionNameToIdMapper {
     }
 
     private static List<JPlagComparison> getComparisons(JPlagResult result) {
-        int numberOfComparisons = result.getOptions().getMaximumNumberOfComparisons();
+        int numberOfComparisons = result.getOptions().maximumNumberOfComparisons();
         return result.getComparisons(numberOfComparisons);
     }
 }

core/src/main/java/de/jplag/reporting/reportobject/model/Cluster.java

@@ -4,6 +4,6 @@ import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 
-public record Cluster(@JsonProperty("average_similarity") float averageSimilarity, @JsonProperty("strength") float strength,
+public record Cluster(@JsonProperty("average_similarity") double averageSimilarity, @JsonProperty("strength") double strength,
         @JsonProperty("members") List<String> members) {
 }

core/src/main/java/de/jplag/reporting/reportobject/model/ComparisonReport.java

@@ -12,6 +12,6 @@ import com.fasterxml.jackson.annotation.JsonProperty;
  * @param matches the list of matches found in the comparison of the two submissions
  */
 public record ComparisonReport(@JsonProperty("id1") String firstSubmissionId, @JsonProperty("id2") String secondSubmissionId,
-        @JsonProperty("similarity") float matchPercentage, @JsonProperty("matches") List<Match> matches) {
+        @JsonProperty("similarity") double matchPercentage, @JsonProperty("matches") List<Match> matches) {
 
 }

core/src/main/java/de/jplag/reporting/reportobject/model/TopComparison.java

@@ -3,5 +3,5 @@ package de.jplag.reporting.reportobject.model;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 public record TopComparison(@JsonProperty("first_submission") String firstSubmission, @JsonProperty("second_submission") String secondSubmission,
-        @JsonProperty("match_percentage") float matchPercentage) {
+        @JsonProperty("match_percentage") double matchPercentage) {
 }

core/src/main/java/de/jplag/strategy/AbstractComparisonStrategy.java

@@ -36,9 +36,8 @@ public abstract class AbstractComparisonStrategy implements ComparisonStrategy {
     protected void compareSubmissionsToBaseCode(SubmissionSet submissionSet) {
         Submission baseCodeSubmission = submissionSet.getBaseCode();
         for (Submission currentSubmission : submissionSet.getSubmissions()) {
-            JPlagComparison baseCodeComparison = greedyStringTiling.compareWithBaseCode(currentSubmission, baseCodeSubmission);
+            JPlagComparison baseCodeComparison = greedyStringTiling.generateBaseCodeMarking(currentSubmission, baseCodeSubmission);
             currentSubmission.setBaseCodeComparison(baseCodeComparison);
-            baseCodeSubmission.resetBaseCode();
         }
     }
 
@@ -49,7 +48,7 @@ public abstract class AbstractComparisonStrategy implements ComparisonStrategy {
         JPlagComparison comparison = greedyStringTiling.compare(first, second);
         logger.info("Comparing {}-{}: {}", first.getName(), second.getName(), comparison.similarity());
 
-        if (options.getSimilarityMetric().isAboveThreshold(comparison, options.getSimilarityThreshold())) {
+        if (options.similarityMetric().isAboveThreshold(comparison, options.similarityThreshold())) {
             return Optional.of(comparison);
         }
         return Optional.empty();

core/src/test/java/de/jplag/BaseCodeTest.java

@@ -16,53 +16,53 @@ public class BaseCodeTest extends TestBase {
 
     @Test
     void testBasecodeUserSubmissionComparison() throws ExitException {
-        JPlagResult result = runJPlag("basecode", it -> it.setBaseCodeSubmissionName("base"));
+        JPlagResult result = runJPlag("basecode", it -> it.withBaseCodeSubmissionName("base"));
         verifyResults(result);
     }
 
     @Test
     void testTinyBasecode() {
-        assertThrows(BasecodeException.class, () -> runJPlag("TinyBasecode", it -> it.setBaseCodeSubmissionName("base")));
+        assertThrows(BasecodeException.class, () -> runJPlag("TinyBasecode", it -> it.withBaseCodeSubmissionName("base")));
     }
 
     @Test
     void testEmptySubmission() throws ExitException {
-        JPlagResult result = runJPlag("emptysubmission", it -> it.setBaseCodeSubmissionName("base"));
+        JPlagResult result = runJPlag("emptysubmission", it -> it.withBaseCodeSubmissionName("base"));
         verifyResults(result);
     }
 
     @Test
     void testAutoTrimFileSeparators() throws ExitException {
-        JPlagResult result = runJPlag("basecode", it -> it.setBaseCodeSubmissionName(File.separator + "base" + File.separator));
+        JPlagResult result = runJPlag("basecode", it -> it.withBaseCodeSubmissionName(File.separator + "base" + File.separator));
         verifyResults(result);
     }
 
     private void verifyResults(JPlagResult result) {
         assertEquals(2, result.getNumberOfSubmissions());
         assertEquals(1, result.getAllComparisons().size());
-        assertEquals(1, result.getAllComparisons().get(0).getMatches().size());
+        assertEquals(1, result.getAllComparisons().get(0).matches().size());
         assertEquals(1, result.getSimilarityDistribution()[1]);
-        assertEquals(85f, result.getAllComparisons().get(0).similarity(), DELTA);
+        assertEquals(85, result.getAllComparisons().get(0).similarity(), DELTA);
     }
 
     @Test
     void testBasecodePathComparison() throws ExitException {
-        JPlagResult result = runJPlag("basecode", it -> it.setBaseCodeSubmissionName(getBasePath("basecode-base")));
+        JPlagResult result = runJPlag("basecode", it -> it.withBaseCodeSubmissionName(getBasePath("basecode-base")));
         assertEquals(3, result.getNumberOfSubmissions()); // "basecode/base" is now a user submission.
     }
 
     @Test
     void testInvalidRoot() {
-        assertThrows(RootDirectoryException.class, () -> runJPlag("basecode", it -> it.setSubmissionDirectories(List.of("WrongRoot"))));
+        assertThrows(RootDirectoryException.class, () -> runJPlag("basecode", it -> it.withSubmissionDirectories(List.of("WrongRoot"))));
     }
 
     @Test
     void testInvalidBasecode() {
-        assertThrows(BasecodeException.class, () -> runJPlag("basecode", it -> it.setBaseCodeSubmissionName("WrongBasecode")));
+        assertThrows(BasecodeException.class, () -> runJPlag("basecode", it -> it.withBaseCodeSubmissionName("WrongBasecode")));
     }
 
     @Test
     void testBasecodeUserSubmissionWithDots() {
-        assertThrows(BasecodeException.class, () -> runJPlag("basecode", it -> it.setBaseCodeSubmissionName("base.ext")));
+        assertThrows(BasecodeException.class, () -> runJPlag("basecode", it -> it.withBaseCodeSubmissionName("base.ext")));
     }
 }

core/src/test/java/de/jplag/InvalidSubmissionTest.java

@@ -22,7 +22,7 @@ class InvalidSubmissionTest extends TestBase {
     @Test
     void testInvalidSubmissionsWithDebug() throws ExitException {
         try {
-            runJPlag(SAMPLE_NAME, it -> it.setDebugParser(true));
+            runJPlag(SAMPLE_NAME, it -> it.withDebugParser(true));
             fail("No submission exception was thrown!");
         } catch (SubmissionException e) {
             System.out.println(e.getMessage());

core/src/test/java/de/jplag/NewJavaFeaturesTest.java

@@ -28,7 +28,7 @@ public class NewJavaFeaturesTest extends TestBase {
         // Check similarity and number of matches:
         var comparison = result.getAllComparisons().get(0);
         assertEquals(EXPECTED_SIMILARITY, comparison.similarity(), DELTA);
-        assertEquals(EXPECTED_MATCHES, comparison.getMatches().size());
+        assertEquals(EXPECTED_MATCHES, comparison.matches().size());
     }
 
 }

core/src/test/java/de/jplag/NormalComparisonTest.java

@@ -23,9 +23,9 @@ class NormalComparisonTest extends TestBase {
 
         assertEquals(2, result.getNumberOfSubmissions());
         assertEquals(1, result.getAllComparisons().size());
-        assertEquals(1, result.getAllComparisons().get(0).getMatches().size());
+        assertEquals(1, result.getAllComparisons().get(0).matches().size());
         assertEquals(1, result.getSimilarityDistribution()[3]);
-        assertEquals(62.07f, result.getAllComparisons().get(0).similarity(), 0.1f);
+        assertEquals(62.07, result.getAllComparisons().get(0).similarity(), 0.1);
     }
 
     /**
@@ -34,13 +34,13 @@ class NormalComparisonTest extends TestBase {
     @Test
     void testWithMinTokenMatch() throws ExitException {
         var expectedDistribution = new int[] {1, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-        JPlagResult result = runJPlag("SimpleDuplicate", it -> it.setMinimumTokenMatch(5));
+        JPlagResult result = runJPlag("SimpleDuplicate", it -> it.withMinimumTokenMatch(5));
 
         assertEquals(2, result.getNumberOfSubmissions());
         assertEquals(1, result.getAllComparisons().size());
-        assertEquals(2, result.getAllComparisons().get(0).getMatches().size());
+        assertEquals(2, result.getAllComparisons().get(0).matches().size());
         assertArrayEquals(expectedDistribution, result.getSimilarityDistribution());
-        assertEquals(96.55f, result.getAllComparisons().get(0).similarity(), 0.1f);
+        assertEquals(96.55, result.getAllComparisons().get(0).similarity(), 0.1);
     }
 
     /**
@@ -53,7 +53,7 @@ class NormalComparisonTest extends TestBase {
         assertEquals(3, result.getNumberOfSubmissions());
         assertEquals(3, result.getAllComparisons().size());
 
-        result.getAllComparisons().forEach(comparison -> assertEquals(0f, comparison.similarity(), 0.1f));
+        result.getAllComparisons().forEach(comparison -> assertEquals(0, comparison.similarity(), 0.1));
     }
 
     /**
@@ -70,42 +70,41 @@ class NormalComparisonTest extends TestBase {
 
         // All comparisons with E shall have no matches
         result.getAllComparisons().stream()
-                .filter(comparison -> comparison.getSecondSubmission().getName().equals("E") || comparison.getFirstSubmission().getName().equals("E"))
-                .forEach(comparison -> assertEquals(0f, comparison.similarity(), DELTA));
+                .filter(comparison -> comparison.secondSubmission().getName().equals("E") || comparison.firstSubmission().getName().equals("E"))
+                .forEach(comparison -> assertEquals(0, comparison.similarity(), DELTA));
 
         // Hard coded assertions on selected comparisons
-        assertEquals(24.6f, getSelectedPercent(result, "A", "B"), 0.1f);
-        assertEquals(99.7f, getSelectedPercent(result, "A", "C"), 0.1f);
-        assertEquals(77.9f, getSelectedPercent(result, "A", "D"), 0.1f);
-        assertEquals(24.6f, getSelectedPercent(result, "B", "C"), 0.1f);
-        assertEquals(28.3f, getSelectedPercent(result, "B", "D"), 0.1f);
-        assertEquals(77.9f, getSelectedPercent(result, "C", "D"), 0.1f);
+        assertEquals(24.6, getSelectedPercent(result, "A", "B"), 0.1);
+        assertEquals(99.7, getSelectedPercent(result, "A", "C"), 0.1);
+        assertEquals(77.9, getSelectedPercent(result, "A", "D"), 0.1);
+        assertEquals(24.6, getSelectedPercent(result, "B", "C"), 0.1);
+        assertEquals(28.3, getSelectedPercent(result, "B", "D"), 0.1);
+        assertEquals(77.9, getSelectedPercent(result, "C", "D"), 0.1);
 
         // More detailed assertions for the plagiarism in A-D
         var biggestMatch = getSelectedComparison(result, "A", "D");
-        assertEquals(96.4f, biggestMatch.get().maximalSimilarity(), 0.1f);
-        assertEquals(65.3f, biggestMatch.get().minimalSimilarity(), 0.1f);
-        assertEquals(12, biggestMatch.get().getMatches().size());
+        assertEquals(96.4, biggestMatch.get().maximalSimilarity(), 0.1);
+        assertEquals(65.3, biggestMatch.get().minimalSimilarity(), 0.1);
+        assertEquals(12, biggestMatch.get().matches().size());
 
     }
 
     // TODO SH: Methods like this should be moved to the API and also should accept wildcards
-    private float getSelectedPercent(JPlagResult result, String nameA, String nameB) {
-        return getSelectedComparison(result, nameA, nameB).map(JPlagComparison::similarity).orElse(-1f);
+    private double getSelectedPercent(JPlagResult result, String nameA, String nameB) {
+        return getSelectedComparison(result, nameA, nameB).map(JPlagComparison::similarity).orElse(-1.0);
     }
 
     private Optional<JPlagComparison> getSelectedComparison(JPlagResult result, String nameA, String nameB) {
-        return result.getAllComparisons().stream().filter(
-                comparison -> comparison.getFirstSubmission().getName().equals(nameA) && comparison.getSecondSubmission().getName().equals(nameB)
-                        || comparison.getFirstSubmission().getName().equals(nameB) && comparison.getSecondSubmission().getName().equals(nameA))
+        return result.getAllComparisons().stream()
+                .filter(comparison -> comparison.firstSubmission().getName().equals(nameA) && comparison.secondSubmission().getName().equals(nameB)
+                        || comparison.firstSubmission().getName().equals(nameB) && comparison.secondSubmission().getName().equals(nameA))
                 .findFirst();
     }
 
     @Test
     void testMultiRootDirNoBasecode() throws ExitException {
         List<String> paths = List.of(getBasePath("basecode"), getBasePath("SimpleDuplicate")); // 3 + 2 submissions.
-        JPlagResult result = runJPlag(paths, options -> {
-        });
+        JPlagResult result = runJPlag(paths, it -> it);
         assertEquals(5, result.getNumberOfSubmissions());
     }
 
@@ -113,7 +112,7 @@ class NormalComparisonTest extends TestBase {
     void testMultiRootDirSeparateBasecode() throws ExitException {
         String basecodePath = getBasePath("basecode-base");
         List<String> paths = List.of(getBasePath("basecode"), getBasePath("SimpleDuplicate")); // 3 + 2 submissions.
-        JPlagResult result = runJPlag(paths, it -> it.setBaseCodeSubmissionName(basecodePath));
+        JPlagResult result = runJPlag(paths, it -> it.withBaseCodeSubmissionName(basecodePath));
         assertEquals(5, result.getNumberOfSubmissions());
     }
 
@@ -121,7 +120,7 @@ class NormalComparisonTest extends TestBase {
     public void testMultiRootDirBasecodeInSubmissionDir() throws ExitException {
         String basecodePath = getBasePath("basecode", "base");
         List<String> paths = List.of(getBasePath("basecode"), getBasePath("SimpleDuplicate")); // 2 + 2 submissions.
-        JPlagResult result = runJPlag(paths, it -> it.setBaseCodeSubmissionName(basecodePath));
+        JPlagResult result = runJPlag(paths, it -> it.withBaseCodeSubmissionName(basecodePath));
         assertEquals(4, result.getNumberOfSubmissions());
     }
 
@@ -129,15 +128,14 @@ class NormalComparisonTest extends TestBase {
     public void testMultiRootDirBasecodeName() {
         List<String> paths = List.of(getBasePath("basecode"), getBasePath("SimpleDuplicate"));
         String basecodePath = "base"; // Should *not* find basecode/base
-        assertThrows(BasecodeException.class, () -> runJPlag(paths, it -> it.setBaseCodeSubmissionName(basecodePath)));
+        assertThrows(BasecodeException.class, () -> runJPlag(paths, it -> it.withBaseCodeSubmissionName(basecodePath)));
     }
 
     @Test
     public void testDisjunctNewAndOldRootDirectories() throws ExitException {
         List<String> newDirectories = List.of(getBasePath("SimpleDuplicate")); // 2 submissions
         List<String> oldDirectories = List.of(getBasePath("basecode")); // 3 submissions
-        JPlagResult result = runJPlag(newDirectories, oldDirectories, it -> {
-        });
+        JPlagResult result = runJPlag(newDirectories, oldDirectories, it -> it);
         int numberOfExpectedComparison = 1 + 3 * 2;
         assertEquals(numberOfExpectedComparison, result.getAllComparisons().size());
     }
@@ -146,8 +144,7 @@ class NormalComparisonTest extends TestBase {
     void testOverlappingNewAndOldDirectoriesOverlap() throws ExitException {
         List<String> newDirectories = List.of(getBasePath("SimpleDuplicate")); // 2 submissions
         List<String> oldDirectories = List.of(getBasePath("SimpleDuplicate"));
-        JPlagResult result = runJPlag(newDirectories, oldDirectories, it -> {
-        });
+        JPlagResult result = runJPlag(newDirectories, oldDirectories, it -> it);
         int numberOfExpectedComparison = 1;
         assertEquals(numberOfExpectedComparison, result.getAllComparisons().size());
     }
@@ -157,7 +154,7 @@ class NormalComparisonTest extends TestBase {
         String basecodePath = getBasePath("basecode", "base");
         List<String> newDirectories = List.of(getBasePath("SimpleDuplicate")); // 2 submissions
         List<String> oldDirectories = List.of(getBasePath("basecode")); // 3 - 1 submissions
-        JPlagResult result = runJPlag(newDirectories, oldDirectories, it -> it.setBaseCodeSubmissionName(basecodePath));
+        JPlagResult result = runJPlag(newDirectories, oldDirectories, it -> it.withBaseCodeSubmissionName(basecodePath));
         int numberOfExpectedComparison = 1 + 2 * 2;
         assertEquals(numberOfExpectedComparison, result.getAllComparisons().size());
     }

core/src/test/java/de/jplag/ParallelComparisonTest.java

@@ -22,13 +22,13 @@ public class ParallelComparisonTest extends TestBase {
      */
     @Test
     public void testSimpleDuplicate() throws ExitException {
-        JPlagResult result = runJPlag("SimpleDuplicate", it -> it.setComparisonMode(PARALLEL));
+        JPlagResult result = runJPlag("SimpleDuplicate", it -> it.withComparisonMode(PARALLEL));
 
         assertEquals(2, result.getNumberOfSubmissions());
         assertEquals(1, result.getAllComparisons().size());
-        assertEquals(1, result.getAllComparisons().get(0).getMatches().size());
+        assertEquals(1, result.getAllComparisons().get(0).matches().size());
         assertEquals(1, result.getSimilarityDistribution()[3]);
-        assertEquals(62.07f, result.getAllComparisons().get(0).similarity(), DELTA);
+        assertEquals(62.07, result.getAllComparisons().get(0).similarity(), DELTA);
     }
 
     /**
@@ -36,12 +36,12 @@ public class ParallelComparisonTest extends TestBase {
      */
     @Test
     public void testNoDuplicate() throws ExitException {
-        JPlagResult result = runJPlag("NoDuplicate", it -> it.setComparisonMode(PARALLEL));
+        JPlagResult result = runJPlag("NoDuplicate", it -> it.withComparisonMode(PARALLEL));
 
         assertEquals(3, result.getNumberOfSubmissions());
         assertEquals(3, result.getAllComparisons().size());
 
-        result.getAllComparisons().forEach(comparison -> assertEquals(0f, comparison.similarity(), DELTA));
+        result.getAllComparisons().forEach(comparison -> assertEquals(0, comparison.similarity(), DELTA));
     }
 
     /**
@@ -51,41 +51,40 @@ public class ParallelComparisonTest extends TestBase {
      */
     @Test
     public void testPartialPlagiarism() throws ExitException {
-        JPlagResult result = runJPlag("PartialPlagiarism", it -> it.setComparisonMode(PARALLEL));
+        JPlagResult result = runJPlag("PartialPlagiarism", it -> it.withComparisonMode(PARALLEL));
 
         assertEquals(5, result.getNumberOfSubmissions());
         assertEquals(10, result.getAllComparisons().size());
 
         // All comparisons with E shall have no matches
         result.getAllComparisons().stream()
-                .filter(comparison -> comparison.getSecondSubmission().getName().equals("E") || comparison.getFirstSubmission().getName().equals("E"))
-                .forEach(comparison -> assertEquals(0f, comparison.similarity(), DELTA));
+                .filter(comparison -> comparison.secondSubmission().getName().equals("E") || comparison.firstSubmission().getName().equals("E"))
+                .forEach(comparison -> assertEquals(0, comparison.similarity(), DELTA));
 
         // Hard coded assertions on selected comparisons
-        assertEquals(24.6f, getSelectedPercent(result, "A", "B"), DELTA);
-        assertEquals(99.7f, getSelectedPercent(result, "A", "C"), DELTA);
-        assertEquals(77.9f, getSelectedPercent(result, "A", "D"), DELTA);
-        assertEquals(24.6f, getSelectedPercent(result, "B", "C"), DELTA);
-        assertEquals(28.3f, getSelectedPercent(result, "B", "D"), DELTA);
-        assertEquals(77.9f, getSelectedPercent(result, "C", "D"), DELTA);
+        assertEquals(24.6, getSelectedPercent(result, "A", "B"), DELTA);
+        assertEquals(99.7, getSelectedPercent(result, "A", "C"), DELTA);
+        assertEquals(77.9, getSelectedPercent(result, "A", "D"), DELTA);
+        assertEquals(24.6, getSelectedPercent(result, "B", "C"), DELTA);
+        assertEquals(28.3, getSelectedPercent(result, "B", "D"), DELTA);
+        assertEquals(77.9, getSelectedPercent(result, "C", "D"), DELTA);
 
         // More detailed assertions for the plagiarism in A-D
         var biggestMatch = getSelectedComparison(result, "A", "D");
-        assertEquals(96.4f, biggestMatch.get().maximalSimilarity(), DELTA);
-        assertEquals(65.3f, biggestMatch.get().minimalSimilarity(), DELTA);
-        assertEquals(12, biggestMatch.get().getMatches().size());
-
+        assertEquals(96.4, biggestMatch.get().maximalSimilarity(), DELTA);
+        assertEquals(65.3, biggestMatch.get().minimalSimilarity(), DELTA);
+        assertEquals(12, biggestMatch.get().matches().size());
     }
 
     // TODO SH: Methods like this should be moved to the API and also should accept wildcards
-    private float getSelectedPercent(JPlagResult result, String nameA, String nameB) {
-        return getSelectedComparison(result, nameA, nameB).map(JPlagComparison::similarity).orElse(-1f);
+    private double getSelectedPercent(JPlagResult result, String nameA, String nameB) {
+        return getSelectedComparison(result, nameA, nameB).map(JPlagComparison::similarity).orElse(-1.0);
     }
 
     private Optional<JPlagComparison> getSelectedComparison(JPlagResult result, String nameA, String nameB) {
-        return result.getAllComparisons().stream().filter(
-                comparison -> comparison.getFirstSubmission().getName().equals(nameA) && comparison.getSecondSubmission().getName().equals(nameB)
-                        || comparison.getFirstSubmission().getName().equals(nameB) && comparison.getSecondSubmission().getName().equals(nameA))
+        return result.getAllComparisons().stream()
+                .filter(comparison -> comparison.firstSubmission().getName().equals(nameA) && comparison.secondSubmission().getName().equals(nameB)
+                        || comparison.firstSubmission().getName().equals(nameB) && comparison.secondSubmission().getName().equals(nameA))
                 .findFirst();
     }
 }

core/src/test/java/de/jplag/TestBase.java

@@ -4,7 +4,7 @@ import java.io.File;
 import java.nio.file.Path;
 import java.util.List;
 import java.util.StringJoiner;
-import java.util.function.Consumer;
+import java.util.function.Function;
 
 import de.jplag.exceptions.ExitException;
 import de.jplag.java.Language;
@@ -14,7 +14,7 @@ import de.jplag.options.Verbosity;
 public abstract class TestBase {
 
     protected static final String BASE_PATH = Path.of("src", "test", "resources", "de", "jplag", "samples").toString();
-    protected static final float DELTA = 0.1f;
+    protected static final double DELTA = 0.1;
 
     protected String getBasePath() {
         return BASE_PATH;
@@ -31,26 +31,26 @@ public abstract class TestBase {
 
     protected JPlagResult runJPlagWithExclusionFile(String testSampleName, String exclusionFileName) throws ExitException {
         String blackList = Path.of(BASE_PATH, testSampleName, exclusionFileName).toString();
-        return runJPlag(testSampleName, options -> options.setExclusionFileName(blackList));
+        return runJPlag(testSampleName, options -> options.withExclusionFileName(blackList));
     }
 
     protected JPlagResult runJPlagWithDefaultOptions(String testSampleName) throws ExitException {
-        return runJPlag(testSampleName, options -> {
-        });
+        return runJPlag(testSampleName, options -> options);
     }
 
-    protected JPlagResult runJPlag(String testSampleName, Consumer<JPlagOptions> customization) throws ExitException {
+    protected JPlagResult runJPlag(String testSampleName, Function<JPlagOptions, JPlagOptions> customization) throws ExitException {
         return runJPlag(List.of(getBasePath(testSampleName)), List.of(), customization);
     }
 
-    protected JPlagResult runJPlag(List<String> newPaths, Consumer<JPlagOptions> customization) throws ExitException {
+    protected JPlagResult runJPlag(List<String> newPaths, Function<JPlagOptions, JPlagOptions> customization) throws ExitException {
         return runJPlag(newPaths, List.of(), customization);
     }
 
-    protected JPlagResult runJPlag(List<String> newPaths, List<String> oldPaths, Consumer<JPlagOptions> customization) throws ExitException {
-        JPlagOptions options = new JPlagOptions(newPaths, oldPaths, Language.IDENTIFIER);
-        options.setVerbosity(Verbosity.LONG);
-        customization.accept(options);
+    protected JPlagResult runJPlag(List<String> newPaths, List<String> oldPaths, Function<JPlagOptions, JPlagOptions> customization)
+            throws ExitException {
+        JPlagOptions options = new JPlagOptions(LanguageLoader.getLanguage(Language.IDENTIFIER).orElseThrow(), newPaths, oldPaths);
+        options = customization.apply(options);
+        options = options.withVerbosity(Verbosity.LONG);
         JPlag jplag = new JPlag(options);
         return jplag.run();
     }

core/src/test/java/de/jplag/clustering/ClusteringAdapterTest.java

@@ -28,8 +28,8 @@ public class ClusteringAdapterTest {
         for (int i = 0; i < submissions.size(); i++) {
             for (int j = i + 1; j < submissions.size(); j++) {
                 JPlagComparison comparison = mock(JPlagComparison.class);
-                when(comparison.getFirstSubmission()).thenReturn(submissions.get(i));
-                when(comparison.getSecondSubmission()).thenReturn(submissions.get(j));
+                when(comparison.firstSubmission()).thenReturn(submissions.get(i));
+                when(comparison.secondSubmission()).thenReturn(submissions.get(j));
                 comparisons.add(comparison);
             }
         }
@@ -41,7 +41,7 @@ public class ClusteringAdapterTest {
             return List.of(IntStream.range(0, arg.getRowDimension()).boxed().collect(Collectors.toList()));
         });
 
-        ClusteringAdapter clustering = new ClusteringAdapter(comparisons, x -> 0.f);
+        ClusteringAdapter clustering = new ClusteringAdapter(comparisons, x -> 0.0);
         ClusteringResult<Submission> clusteringResult = clustering.doClustering(algorithm);
 
         Collection<Collection<Submission>> expectedResult = List.of(submissions);

core/src/test/java/de/jplag/clustering/ClusteringResultTest.java

@@ -16,8 +16,8 @@ class ClusteringResultTest {
         RealMatrix similarity = new Array2DRowRealMatrix(4, 4);
 
         // These are similar
-        setEntries(similarity, 0, 1, 1f);
-        setEntries(similarity, 2, 3, 1f);
+        setEntries(similarity, 0, 1, 1.0);
+        setEntries(similarity, 2, 3, 1.0);
 
         // Others are dissimilar
 
@@ -36,14 +36,14 @@ class ClusteringResultTest {
         // cluster
 
         // These are similar
-        setEntries(similarity, 0, 1, 0.1f);
-        setEntries(similarity, 2, 3, 0.1f);
+        setEntries(similarity, 0, 1, 0.1);
+        setEntries(similarity, 2, 3, 0.1);
 
         // Others are dissimilar
-        setEntries(similarity, 0, 2, 0.05f);
-        setEntries(similarity, 0, 3, 0.05f);
-        setEntries(similarity, 1, 2, 0.05f);
-        setEntries(similarity, 1, 3, 0.05f);
+        setEntries(similarity, 0, 2, 0.05);
+        setEntries(similarity, 0, 3, 0.05);
+        setEntries(similarity, 1, 2, 0.05);
+        setEntries(similarity, 1, 3, 0.05);
 
         ClusteringResult<Integer> result = ClusteringResult.fromIntegerCollections(List.of(List.of(0, 1), List.of(2, 3)), similarity);
 
@@ -95,14 +95,14 @@ class ClusteringResultTest {
         RealMatrix similarity = new Array2DRowRealMatrix(4, 4);
 
         // These are similar
-        setEntries(similarity, 0, 1, 1f);
-        setEntries(similarity, 2, 3, 1f);
+        setEntries(similarity, 0, 1, 1.0);
+        setEntries(similarity, 2, 3, 1.0);
 
         // Others are dissimilar
 
         ClusteringResult<Integer> result = ClusteringResult.fromIntegerCollections(List.of(List.of(0, 1), List.of(2, 3)), similarity);
         var cluster = result.getClusters().stream().findFirst().orElseThrow();
-        assertEquals(1f, cluster.getAverageSimilarity(), 0.00001);
+        assertEquals(1.0, cluster.getAverageSimilarity(), 0.00001);
     }
 
     private static void setEntries(RealMatrix matrix, int i, int j, double similarity) {

core/src/test/java/de/jplag/clustering/algorithm/ClusteringData.java

@@ -23,26 +23,26 @@ public enum ClusteringData {
         }
         // These are similar
         setEntries(similarity, 0, 1, 0.5);
-        setEntries(similarity, 2, 3, 0.5f);
+        setEntries(similarity, 2, 3, 0.5);
 
         // Others are dissimilar
-        setEntries(similarity, 0, 2, 0.1f);
-        setEntries(similarity, 0, 3, 0.1f);
-        setEntries(similarity, 1, 2, 0.1f);
-        setEntries(similarity, 1, 3, 0.1f);
+        setEntries(similarity, 0, 2, 0.1);
+        setEntries(similarity, 0, 3, 0.1);
+        setEntries(similarity, 1, 2, 0.1);
+        setEntries(similarity, 1, 3, 0.1);
 
         return similarity;
-    }, new int[][] {{0, 1}, {2, 3}}, new ClusteringOptions.Builder().agglomerativeThreshold(0.4f));
+    }, new int[][] {{0, 1}, {2, 3}}, new ClusteringOptions().withAgglomerativeThreshold(0.4));
 
     private final RealMatrix similarity;
     private final Set<Set<Integer>> expected;
     private final ClusteringOptions options;
 
-    ClusteringData(Supplier<RealMatrix> similarity, int[][] expected, ClusteringOptions.Builder options) {
+    ClusteringData(Supplier<RealMatrix> similarity, int[][] expected, ClusteringOptions options) {
         this.similarity = similarity.get();
         this.expected = makeSets(
                 Arrays.stream(expected).map(intArray -> Arrays.stream(intArray).boxed().collect(Collectors.toList())).collect(Collectors.toList()));
-        this.options = options.build();
+        this.options = options;
     }
 
     public ClusteringOptions getOptions() {

core/src/test/java/de/jplag/clustering/preprocessors/PercentilePreprocessorTest.java

@@ -1,6 +1,7 @@
 package de.jplag.clustering.preprocessors;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.util.Optional;
 
@@ -9,13 +10,11 @@ import org.junit.jupiter.api.Test;
 
 public class PercentilePreprocessorTest extends PreprocessingTestBase {
 
-    private static final double EPSILON = 0.0000001;
-
     PercentileThresholdProcessor preprocessor;
 
     @BeforeEach
     public void init() {
-        preprocessor = new PercentileThresholdProcessor(0.5f);
+        preprocessor = new PercentileThresholdProcessor(0.5);
     }
 
     @Test
@@ -30,12 +29,11 @@ public class PercentilePreprocessorTest extends PreprocessingTestBase {
         double[][] original = createTestData();
         double[][] result = preprocessor.preprocessSimilarities(original);
         withAllValues(preprocessor, original, result, (originalValue, preprocessed) -> {
-            // Values are only 0.1 and 0.5; percentile 0.5 should only preserve values of
-            // 0.5.
-            if (originalValue > 0.1) {
+            // Median is 0.1 => Values >= 0.1 should preserved.
+            if (originalValue >= 0.1) {
                 assertEquals(Optional.of(originalValue), preprocessed);
             } else {
-                assertEquals(0.0, preprocessed.orElse(0.0), EPSILON);
+                assertTrue(preprocessed.isEmpty());
             }
         });
     }

core/src/test/java/de/jplag/clustering/preprocessors/PreprocessingTestBase.java

@@ -3,7 +3,11 @@ package de.jplag.clustering.preprocessors;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-import java.util.*;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.function.BiConsumer;
 import java.util.function.IntUnaryOperator;
 import java.util.stream.Collectors;
@@ -26,13 +30,13 @@ public class PreprocessingTestBase {
         }
         // These are similar
         setEntries(similarity, 0, 1, 0.5);
-        setEntries(similarity, 2, 3, 0.5f);
+        setEntries(similarity, 2, 3, 0.5);
 
         // Others are dissimilar
-        setEntries(similarity, 0, 2, 0.1f);
-        setEntries(similarity, 0, 3, 0.1f);
-        setEntries(similarity, 1, 2, 0.1f);
-        setEntries(similarity, 1, 3, 0.1f);
+        setEntries(similarity, 0, 2, 0.1);
+        setEntries(similarity, 0, 3, 0.1);
+        setEntries(similarity, 1, 2, 0.1);
+        setEntries(similarity, 1, 3, 0.1);
 
         // last row is empty
 

core/src/test/java/de/jplag/reporting/reportobject/mapper/ClusteringResultMapperTest.java

@@ -21,21 +21,21 @@ public class ClusteringResultMapperTest {
     public void test() {
         // given
         JPlagResult resultMock = mock(JPlagResult.class);
-        Cluster<Submission> cluster1 = createClusterWith(0.2f, 0.4f, "1", "2");
-        Cluster<Submission> cluster2 = createClusterWith(0.3f, 0.6f, "3", "4", "5");
-        when(resultMock.getClusteringResult()).thenReturn(List.of(new ClusteringResult<>(List.of(cluster1, cluster2), 0.3f)));
+        Cluster<Submission> cluster1 = createClusterWith(0.2, 0.4, "1", "2");
+        Cluster<Submission> cluster2 = createClusterWith(0.3, 0.6, "3", "4", "5");
+        when(resultMock.getClusteringResult()).thenReturn(List.of(new ClusteringResult<>(List.of(cluster1, cluster2), 0.3)));
 
         // when
         var result = clusteringResultMapper.map(resultMock);
 
         // then
-        assertEquals(List.of(new de.jplag.reporting.reportobject.model.Cluster(0.4f, 0.2f, List.of("1", "2")),
-                new de.jplag.reporting.reportobject.model.Cluster(0.6f, 0.3f, List.of("3", "4", "5"))
+        assertEquals(List.of(new de.jplag.reporting.reportobject.model.Cluster(0.4, 0.2, List.of("1", "2")),
+                new de.jplag.reporting.reportobject.model.Cluster(0.6, 0.3, List.of("3", "4", "5"))
 
         ), result);
     }
 
-    private Cluster<Submission> createClusterWith(Float communityStrength, Float averageSimilarity, String... ids) {
+    private Cluster<Submission> createClusterWith(Double communityStrength, Double averageSimilarity, String... ids) {
         var submissions = Arrays.stream(ids).map(this::submissionWithId).toList();
         return new Cluster<>(submissions, communityStrength, averageSimilarity);
     }

core/src/test/java/de/jplag/reporting/reportobject/mapper/MetricMapperTest.java

@@ -1,6 +1,8 @@
 package de.jplag.reporting.reportobject.mapper;
 
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.anyInt;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -21,14 +23,14 @@ public class MetricMapperTest {
     public void test_getAverageMetric() {
         // given
         JPlagResult jPlagResult = createJPlagResult(MockMetric.AVG, distribution(2, 3, 5, 7, 11, 13, 17, 19, 23, 29),
-                comparison(submission("1"), submission("2"), .7f), comparison(submission("3"), submission("4"), .3f));
+                comparison(submission("1"), submission("2"), .7), comparison(submission("3"), submission("4"), .3));
         // when
         var result = metricMapper.getAverageMetric(jPlagResult);
 
         // then
         Assertions.assertEquals("AVG", result.name());
         Assertions.assertIterableEquals(List.of(2, 3, 5, 7, 11, 13, 17, 19, 23, 29), result.distribution());
-        Assertions.assertEquals(List.of(new TopComparison("1", "2", .7f), new TopComparison("3", "4", .3f)), result.topComparisons());
+        Assertions.assertEquals(List.of(new TopComparison("1", "2", .7), new TopComparison("3", "4", .3)), result.topComparisons());
         Assertions.assertEquals(
                 "Average of both program coverages. This is the default similarity which"
                         + " works in most cases: Matches with a high average similarity indicate that the programs work " + "in a very similar way.",
@@ -39,14 +41,14 @@ public class MetricMapperTest {
     public void test_getMaxMetric() {
         // given
         JPlagResult jPlagResult = createJPlagResult(MockMetric.MAX, distribution(2, 3, 5, 7, 11, 13, 17, 19, 23, 29),
-                comparison(submission("00"), submission("01"), .7f), comparison(submission("10"), submission("11"), .3f));
+                comparison(submission("00"), submission("01"), .7), comparison(submission("10"), submission("11"), .3));
         // when
         var result = metricMapper.getMaxMetric(jPlagResult);
 
         // then
         Assertions.assertEquals("MAX", result.name());
         Assertions.assertIterableEquals(List.of(2, 3, 5, 7, 11, 13, 17, 19, 23, 29), result.distribution());
-        Assertions.assertEquals(List.of(new TopComparison("00", "01", .7f), new TopComparison("10", "11", .3f)), result.topComparisons());
+        Assertions.assertEquals(List.of(new TopComparison("00", "01", .7), new TopComparison("10", "11", .3)), result.topComparisons());
         Assertions.assertEquals(
                 "Maximum of both program coverages. This ranking is especially useful if the programs are very "
                         + "different in size. This can happen when dead code was inserted to disguise the origin of the plagiarized program.",
@@ -61,7 +63,7 @@ public class MetricMapperTest {
         return new CreateSubmission(name);
     }
 
-    private Comparison comparison(CreateSubmission submission1, CreateSubmission submission2, float similarity) {
+    private Comparison comparison(CreateSubmission submission1, CreateSubmission submission2, double similarity) {
         return new Comparison(submission1, submission2, similarity);
     }
 
@@ -76,7 +78,7 @@ public class MetricMapperTest {
         }
 
         JPlagOptions options = mock(JPlagOptions.class);
-        doReturn(createComparisonsDto.length).when(options).getMaximumNumberOfComparisons();
+        doReturn(createComparisonsDto.length).when(options).maximumNumberOfComparisons();
         doReturn(options).when(jPlagResult).getOptions();
 
         List<JPlagComparison> comparisonList = new ArrayList<>();
@@ -87,8 +89,8 @@ public class MetricMapperTest {
             doReturn(comparisonDto.submission2.name).when(submission2).getName();
 
             JPlagComparison mockedComparison = mock(JPlagComparison.class);
-            doReturn(submission1).when(mockedComparison).getFirstSubmission();
-            doReturn(submission2).when(mockedComparison).getSecondSubmission();
+            doReturn(submission1).when(mockedComparison).firstSubmission();
+            doReturn(submission2).when(mockedComparison).secondSubmission();
             if (metricToMock.equals(MockMetric.AVG)) {
                 doReturn(comparisonDto.similarity).when(mockedComparison).similarity();
             } else if (metricToMock.equals(MockMetric.MAX)) {
@@ -106,7 +108,7 @@ public class MetricMapperTest {
         AVG
     }
 
-    private record Comparison(CreateSubmission submission1, CreateSubmission submission2, float similarity) {
+    private record Comparison(CreateSubmission submission1, CreateSubmission submission2, double similarity) {
     }
 
     private record CreateSubmission(String name) {

core/src/test/java/de/jplag/special/TokenPrinterTest.java

@@ -2,12 +2,16 @@ package de.jplag.special;
 
 import static org.junit.jupiter.api.Assertions.fail;
 
-import java.util.function.Consumer;
+import java.util.function.Function;
 
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
-import de.jplag.*;
+import de.jplag.JPlagResult;
+import de.jplag.LanguageLoader;
+import de.jplag.Submission;
+import de.jplag.TestBase;
+import de.jplag.TokenPrinter;
 import de.jplag.exceptions.ExitException;
 import de.jplag.options.JPlagOptions;
 
@@ -31,42 +35,36 @@ class TokenPrinterTest extends TestBase {
     @Disabled("Not a meaningful test, used for designing the token set")
     @Test
     void printCPPFiles() {
-        printSubmissions(options -> {
-            options.setLanguage(LanguageLoader.getLanguage(LANGUAGE_CPP).orElseThrow());
-            options.setMinimumTokenMatch(MIN_TOKEN_MATCH); // for printing also allow small files
-        });
+        printSubmissions(
+                options -> options.withLanguageOption(LanguageLoader.getLanguage(LANGUAGE_CPP).orElseThrow()).withMinimumTokenMatch(MIN_TOKEN_MATCH));
     }
 
     @Disabled("Not a meaningful test, used for designing the token set")
     @Test
     void printJavaFiles() {
-        printSubmissions(options -> {
-            options.setMinimumTokenMatch(MIN_TOKEN_MATCH); // for printing also allow small files
-        });
+        printSubmissions(options -> options.withMinimumTokenMatch(MIN_TOKEN_MATCH));
     }
 
     @Disabled("Not a meaningful test, used for designing the token set")
     @Test
     void printRLangFiles() {
-        printSubmissions(options -> {
-            options.setLanguage(LanguageLoader.getLanguage(LANGUAGE_R).orElseThrow());
-            options.setMinimumTokenMatch(MIN_TOKEN_MATCH); // for printing also allow small files
-        });
+        printSubmissions(
+                options -> options.withLanguageOption(LanguageLoader.getLanguage(LANGUAGE_R).orElseThrow()).withMinimumTokenMatch(MIN_TOKEN_MATCH));
     }
 
     @Disabled("Not a meaningful test, used for designing the token set")
     @Test
     void printGoFiles() {
-        printSubmissions(options -> options.setLanguage(LanguageLoader.getLanguage(LANGUAGE_GO).orElseThrow()));
+        printSubmissions(options -> options.withLanguageOption(LanguageLoader.getLanguage(LANGUAGE_GO).orElseThrow()));
     }
 
     @Disabled("Not a meaningful test, used for designing the token set")
     @Test
     void printKotlinFiles() {
-        printSubmissions(options -> options.setLanguage(LanguageLoader.getLanguage(LANGUAGE_KOTLIN).orElseThrow()));
+        printSubmissions(options -> options.withLanguageOption(LanguageLoader.getLanguage(LANGUAGE_KOTLIN).orElseThrow()));
     }
 
-    private void printSubmissions(Consumer<JPlagOptions> optionsCustomization) {
+    private void printSubmissions(Function<JPlagOptions, JPlagOptions> optionsCustomization) {
         try {
             JPlagResult result = runJPlag(PRINTER_FOLDER, optionsCustomization);
             for (Submission submission : result.getSubmissions().getSubmissions()) {

core/src/test/java/de/jplag/special/VolumeTest.java

@@ -47,7 +47,7 @@ public class VolumeTest extends TestBase {
             return;
         }
 
-        var results = runJPlag("data", jPlagOptions -> jPlagOptions.setMaximumNumberOfComparisons(-1));
+        var results = runJPlag("data", jPlagOptions -> jPlagOptions.withMaximumNumberOfComparisons(JPlagOptions.SHOW_ALL_COMPARISONS));
 
         var csv = readCSVResults(String.format("%s/%s", this.getBasePath(), "matches_avg.csv"));
 
@@ -55,17 +55,17 @@ public class VolumeTest extends TestBase {
         System.out.println("Volume test size: " + csv.size());
 
         results.getAllComparisons().forEach(result -> {
-            var key = result.getFirstSubmission().getName() + result.getSecondSubmission().getName();
+            var key = result.firstSubmission().getName() + result.secondSubmission().getName();
 
             assertTrue(csv.containsKey(key));
-            assertEquals(csv.getOrDefault(key, -1f), result.similarity(), DELTA);
+            assertEquals(csv.getOrDefault(key, -1.0), result.similarity(), DELTA);
         });
 
     }
 
-    private Map<String, Float> readCSVResults(String filePathAndName) throws IOException {
+    private Map<String, Double> readCSVResults(String filePathAndName) throws IOException {
         List<String> lines = Files.readAllLines(Path.of(filePathAndName), JPlagOptions.CHARSET);
-        var results = new HashMap<String, Float>();
+        var results = new HashMap<String, Double>();
 
         lines.forEach(line -> {
             var entries = line.split(";");
@@ -74,7 +74,7 @@ public class VolumeTest extends TestBase {
                 throw new IllegalArgumentException(String.format("Illegal line: '%s'", line));
             }
 
-            results.put(entries[1] + entries[2], Float.parseFloat((entries[3])));
+            results.put(entries[1] + entries[2], Double.parseDouble((entries[3])));
         });
 
         return results;

endtoend-testing/pom.xml

@@ -30,7 +30,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.13.3</version>
+            <version>2.13.4</version>
         </dependency>
     </dependencies>
 </project>

endtoend-testing/src/main/java/de/jplag/endtoend/helper/TestSuiteHelper.java

@@ -56,7 +56,7 @@ public class TestSuiteHelper {
      */
     public static String getTestIdentifier(JPlagComparison jPlagComparison) {
 
-        return List.of(jPlagComparison.getFirstSubmission(), jPlagComparison.getSecondSubmission()).stream().map(Submission::getFiles)
+        return List.of(jPlagComparison.firstSubmission(), jPlagComparison.secondSubmission()).stream().map(Submission::getFiles)
                 .map(FileHelper::getEnclosedFileNamesFromCollection).sorted().collect(Collectors.joining("-"));
 
     }

endtoend-testing/src/main/java/de/jplag/endtoend/model/ExpectedResult.java

@@ -7,6 +7,6 @@ import com.fasterxml.jackson.annotation.JsonProperty;
  * which can be found at https://github.com/jplag/JPlag/issues/548 Here this object is used for serialization and
  * deserialization of the information from json to object or object to json.
  */
-public record ExpectedResult(@JsonProperty("minimal_similarity") float resultSimilarityMinimum,
-        @JsonProperty("maximum_similarity") float resultSimilarityMaximum, @JsonProperty("matched_token_number") int resultMatchedTokenNumber) {
+public record ExpectedResult(@JsonProperty("minimal_similarity") double resultSimilarityMinimum,
+        @JsonProperty("maximum_similarity") double resultSimilarityMaximum, @JsonProperty("matched_token_number") int resultMatchedTokenNumber) {
 }

endtoend-testing/src/test/java/de/jplag/endtoend/EndToEndSuiteTest.java

@@ -22,10 +22,13 @@ import org.junit.jupiter.api.TestFactory;
 import de.jplag.JPlag;
 import de.jplag.JPlagComparison;
 import de.jplag.JPlagResult;
+import de.jplag.LanguageLoader;
 import de.jplag.endtoend.helper.FileHelper;
 import de.jplag.endtoend.helper.JsonHelper;
 import de.jplag.endtoend.helper.TestSuiteHelper;
-import de.jplag.endtoend.model.*;
+import de.jplag.endtoend.model.ExpectedResult;
+import de.jplag.endtoend.model.Options;
+import de.jplag.endtoend.model.ResultDescription;
 import de.jplag.exceptions.ExitException;
 import de.jplag.options.JPlagOptions;
 
@@ -35,8 +38,9 @@ import de.jplag.options.JPlagOptions;
  * original class. The results are compared with the results from previous tests and changes are detected.
  */
 public class EndToEndSuiteTest {
+    private static final double EPSILON = 1E-8;
     // Language -> directory names and Paths
-    private Map<String, Map<String, Path>> LanguageToTestCaseMapper;
+    private Map<String, Map<String, Path>> languageToTestCaseMapper;
 
     private List<Options> options;
 
@@ -46,7 +50,7 @@ public class EndToEndSuiteTest {
 
     public EndToEndSuiteTest() throws IOException {
         // Loading the test resources
-        LanguageToTestCaseMapper = TestSuiteHelper.getAllLanguageResources();
+        languageToTestCaseMapper = TestSuiteHelper.getAllLanguageResources();
         // creating the temporary lists for the test run
         validationErrors = new ArrayList<>();
         temporaryResultList = new HashMap<>();
@@ -81,7 +85,7 @@ public class EndToEndSuiteTest {
      */
     @TestFactory
     Collection<DynamicTest> dynamicOverAllTest() throws IOException {
-        for (Entry<String, Map<String, Path>> languageMap : LanguageToTestCaseMapper.entrySet()) {
+        for (Entry<String, Map<String, Path>> languageMap : languageToTestCaseMapper.entrySet()) {
             String currentLanguageIdentifier = languageMap.getKey();
             for (Entry<String, Path> languagePaths : languageMap.getValue().entrySet()) {
                 String[] fileNames = FileHelper.loadAllTestFileNames(languagePaths.getValue());
@@ -108,7 +112,7 @@ public class EndToEndSuiteTest {
     /**
      * Superordinate test function to be able to continue to check all data to be tested in case of failed tests
      * @param directoryName name of the current tested directory
-     * @param options for the current test run
+     * @param option for the current test run
      * @param currentLanguageIdentifier current JPlag language option
      * @param testFiles files to be tested
      * @param currentResultDescription results stored for the test data
@@ -142,10 +146,9 @@ public class EndToEndSuiteTest {
             ResultDescription currentResultDescription) throws IOException, ExitException {
         String[] submissionPath = FileHelper.createNewTestCaseDirectory(testFiles);
 
-        JPlagOptions jplagOptions = new JPlagOptions(Arrays.asList(submissionPath), new ArrayList<>(), languageIdentifier);
-
-        jplagOptions.setMinimumTokenMatch(options.minimumTokenMatch());
-
+        var language = LanguageLoader.getLanguage(languageIdentifier).orElseThrow();
+        JPlagOptions jplagOptions = new JPlagOptions(language, Arrays.asList(submissionPath), new ArrayList<>())
+                .withMinimumTokenMatch(options.minimumTokenMatch());
         JPlagResult jplagResult = new JPlag(jplagOptions).run();
 
         List<JPlagComparison> currentJPlagComparison = jplagResult.getAllComparisons();
@@ -154,20 +157,20 @@ public class EndToEndSuiteTest {
             String identifier = TestSuiteHelper.getTestIdentifier(jPlagComparison);
             addToTemporaryResultMap(directoryName, options, jPlagComparison, languageIdentifier);
 
-            assertNotNull(currentResultDescription, "No stored result could be found for the current LanguageOption! " + options.toString());
+            assertNotNull(currentResultDescription, "No stored result could be found for the current LanguageOption! " + options);
 
             ExpectedResult result = currentResultDescription.getExpectedResultByIdentifier(TestSuiteHelper.getTestIdentifier(jPlagComparison));
             assertNotNull(result, "No stored result could be found for the identifier! " + identifier);
 
-            if (Float.compare(result.resultSimilarityMinimum(), jPlagComparison.minimalSimilarity()) != 0) {
+            if (areDoublesDifferent(result.resultSimilarityMinimum(), jPlagComparison.minimalSimilarity())) {
                 addToValidationErrors("minimalSimilarity", String.valueOf(result.resultSimilarityMinimum()),
                         String.valueOf(jPlagComparison.minimalSimilarity()));
             }
-            if (Float.compare(result.resultSimilarityMaximum(), jPlagComparison.maximalSimilarity()) != 0) {
+            if (areDoublesDifferent(result.resultSimilarityMaximum(), jPlagComparison.maximalSimilarity())) {
                 addToValidationErrors("maximalSimilarity", String.valueOf(result.resultSimilarityMaximum()),
                         String.valueOf(jPlagComparison.maximalSimilarity()));
             }
-            if (Integer.compare(result.resultMatchedTokenNumber(), jPlagComparison.getNumberOfMatchedTokens()) != 0) {
+            if (result.resultMatchedTokenNumber() != jPlagComparison.getNumberOfMatchedTokens()) {
                 addToValidationErrors("numberOfMatchedTokens", String.valueOf(result.resultMatchedTokenNumber()),
                         String.valueOf(jPlagComparison.getNumberOfMatchedTokens()));
             }
@@ -176,6 +179,10 @@ public class EndToEndSuiteTest {
         }
     }
 
+    private boolean areDoublesDifferent(double d1, double d2) {
+        return Math.abs(d1 - d2) >= EPSILON;
+    }
+
     /**
      * Creates the display message for failed tests
      * @param valueName Name of the failed test object

languages.api/src/main/java/de/jplag/Language.java

@@ -1,6 +1,7 @@
 package de.jplag;
 
 import java.io.File;
+import java.util.List;
 
 /**
  * Common interface for all languages. Each language-front end must provide a concrete language implementation.
@@ -33,7 +34,7 @@ public interface Language {
      * @param files are the names of the files to parse.
      * @return the list of parsed JPlag tokens.
      */
-    TokenList parse(File directory, String[] files);
+    List<Token> parse(File directory, String[] files);
 
     /**
      * Whether errors were found during the last {@link #parse}.

languages.api/src/main/java/de/jplag/LanguageLoader.java

@@ -36,9 +36,10 @@ public final class LanguageLoader {
                 languages.remove(languageIdentifier);
                 continue;
             }
-            logger.info("Loading Language Frontend '{}'", language.getName());
+            logger.debug("Loading Language Frontend '{}'", language.getName());
             languages.put(languageIdentifier, language);
         }
+        logger.info("Available languages: '{}'", languages.values().stream().map(Language::getName).toList());
 
         cachedLanguageInstances = Collections.unmodifiableMap(languages);
         return cachedLanguageInstances;

languages.api/src/main/java/de/jplag/Token.java

@@ -1,32 +1,36 @@
 package de.jplag;
 
 /**
- * This class represents a token in a source code. It can represents keywords, identifies, syntactical structures etc.
+ * This class represents a token in a source code. It can represent keywords, identifiers, syntactical structures etc.
  * What types of tokens there are depends on the specific language, meaning JPlag does not enforce a specific token set.
  * The language parsers decide what is a token and what is not.
  */
 public abstract class Token {
+    /** Indicates that the requested field has no value. */
+    public static final int NO_VALUE = -1;
+
     private int line;
     private int column;
     private int length;
     private String file;
 
-    private boolean marked;
-    private boolean basecode = false;
-    private int hash = -1; // hash-value. set and used by main algorithm (GSTiling)
-
     protected int type;
 
     /**
      * Creates a token without information about the column or the length of the token in the line.
      * @param type is the token type.
      * @param file is the name of the source code file.
-     * @param line is the line index in the source code where the token resides. Cannot be smaller than 1.
+     * @param line is the line index in the source code where the token resides. Cannot be smaller than 1. For
+     * {@link TokenConstants#FILE_END FILE_END} it is automatically set to {@link #NO_VALUE}.
      */
     public Token(int type, String file, int line) {
         this.type = type;
         this.file = file;
-        setLine(line > 0 ? line : 1);
+        if (type == TokenConstants.FILE_END) {
+            this.line = NO_VALUE;
+        } else {
+            this.line = line > 0 ? line : 1;
+        }
     }
 
     /**
@@ -89,22 +93,6 @@ public abstract class Token {
         this.column = column;
     }
 
-    /**
-     * Sets the length if the code sections represented by this token.
-     * @param length is the length in characters to set.
-     */
-    public void setLength(int length) {
-        this.length = length;
-    }
-
-    /**
-     * Sets the line index denoting in which line the code sections represented by this token starts.
-     * @param line is the line index to set.
-     */
-    public void setLine(int line) {
-        this.line = line;
-    }
-
     @Override
     public String toString() {
         return type2string();
@@ -114,40 +102,4 @@ public abstract class Token {
      * @return a string representation depending on the type of the token.
      */
     protected abstract String type2string();
-
-    /* package-private */ int getHash() {
-        return hash;
-    }
-
-    /**
-     * @return whether this token is part of a basecode.
-     */
-    /* package-private */ boolean isBasecode() {
-        return basecode;
-    }
-
-    /**
-     * @return whether this token is marked by the comparison algorithm.
-     */
-    /* package-private */ boolean isMarked() {
-        return marked;
-    }
-
-    /* package-private */ boolean setBasecode(boolean basecode) {
-        this.basecode = basecode;
-        return basecode;
-    }
-
-    /* package-private */ void setFile(String file) {
-        this.file = file;
-    }
-
-    /* package-private */ void setHash(int hash) {
-        this.hash = hash;
-    }
-
-    /* package-private */ boolean setMarked(boolean marked) {
-        this.marked = marked;
-        return marked;
-    }
 }

languages.api/src/main/java/de/jplag/TokenHashMap.java

@@ -1,77 +0,0 @@
-package de.jplag;
-
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A {@link HashMap} that maps Integer keys to multiple Integer values. Note that all keys with identical
- * <code>(key % prime)</code> are mapped to the same values. Specifically, <code>prime</code> is the next prime number
- * that is larger or equal to the specified size (see {@link TokenHashMap#TokenHashMap(int)}).
- */
-public class TokenHashMap {
-    private static final int CERTAINTY = 100;
-    private final Map<Integer, List<Integer>> mappedEntries;
-    private final int primeNumber;
-
-    /**
-     * Creates the {@link HashMap}.
-     * @param size specifies the initial size and the key mapping (see {@link TokenHashMap}).
-     */
-    public TokenHashMap(int size) {
-        mappedEntries = new HashMap<>(size);
-        primeNumber = nextPrimeNumber(size);
-    }
-
-    /**
-     * Returns all stored numbers for a key. Note that all keys with identical <code>(key % prime)</code> are mapped to the
-     * same values (see {@link TokenHashMap}).
-     * @param key is the specific key.
-     * @return the stored numbers or an empty list if nothing is stored.
-     */
-    public final List<Integer> get(int key) {
-        int actualKey = key % primeNumber;
-        if (mappedEntries.containsKey(actualKey)) {
-            return new ArrayList<>(mappedEntries.get(actualKey));
-        }
-        return Collections.emptyList();
-    }
-
-    /**
-     * Stores a number for a key, does not replace the previous stored numbers for that key. Note that all keys with
-     * identical <code>(key % prime)</code> are mapped to the same values (see {@link TokenHashMap}).
-     * @param key is the specific key.
-     * @param value is the number to store.
-     */
-    public final void put(int key, int value) {
-        int actualKey = key % primeNumber;
-        if (mappedEntries.containsKey(actualKey)) {
-            mappedEntries.get(actualKey).add(value);
-        } else {
-            List<Integer> entries = new ArrayList<>();
-            entries.add(value);
-            mappedEntries.put(actualKey, entries);
-        }
-    }
-
-    /**
-     * Calculates the next prime number (including 1) that is larger or equal to a given number.
-     * @param number is the give number.
-     * @return the next prime number.
-     */
-    private int nextPrimeNumber(int number) {
-        if (number <= 1) {
-            return 1;
-        }
-        for (int possiblePrime = number; possiblePrime < 2 * number; possiblePrime++) { // Bertrand's postulate
-            BigInteger bigInt = BigInteger.valueOf(possiblePrime);
-            if (bigInt.isProbablePrime(CERTAINTY)) {
-                return possiblePrime;
-            }
-        }
-        throw new IllegalStateException("Should never be reached because of Bertrand's postulate!");
-    }
-}

languages.api/src/main/java/de/jplag/TokenList.java

@@ -1,76 +0,0 @@
-package de.jplag;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-/**
- * List of tokens. Allows random access to individual tokens. Contains a hash map for token hashes.
- */
-public class TokenList {
-    private final List<Token> tokens;
-    TokenHashMap tokenHashes = null;
-    int hashLength = -1;
-
-    /**
-     * Creates an empty token list.
-     */
-    public TokenList() {
-        tokens = new ArrayList<>();
-    }
-
-    /**
-     * @return the number of tokens in the list.
-     */
-    public final int size() {
-        return tokens.size();
-    }
-
-    /**
-     * Adds an token to the list.
-     * @param token is the token to add.
-     */
-    public final void addToken(Token token) {
-        if (tokens.size() > 0) {
-            Token lastToken = tokens.get(tokens.size() - 1);
-            if (lastToken.getFile().equals(token.getFile())) {
-                token.setFile(lastToken.getFile()); // To save memory ...
-            }
-            if (token.getLine() < lastToken.getLine() && (token.getFile().equals(lastToken.getFile()))) {
-                token.setLine(lastToken.getLine()); // just to make sure
-            }
-        }
-        tokens.add(token);
-    }
-
-    /**
-     * Returns a view on all tokens.
-     * @return all tokens.
-     */
-    public Collection<Token> allTokens() {
-        return new ArrayList<>(tokens);
-    }
-
-    /**
-     * Grants access to a specific token.
-     * @param index is the token index.
-     * @return the desired token.
-     * @throws IllegalArgumentException if the index is out of bounds.
-     */
-    public Token getToken(int index) {
-        if (index < 0 || index >= tokens.size()) {
-            throw new IllegalArgumentException("Cannot access token with index " + index + ", there are only " + tokens.size() + " tokens!");
-        }
-        return tokens.get(index);
-    }
-
-    @Override
-    public final String toString() {
-        try {
-            List<String> tokenStrings = tokens.stream().map(Token::toString).toList();
-            return String.join(System.lineSeparator(), tokenStrings);
-        } catch (OutOfMemoryError exception) {
-            return "Token list to large for output: " + tokens.size() + " Tokens";
-        }
-    }
-}

languages.api/src/main/java/de/jplag/TokenPrinter.java

@@ -4,7 +4,9 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -46,22 +48,22 @@ public final class TokenPrinter {
 
     /**
      * Creates a string representation of a set of files line by line and adds the tokens under the lines.
-     * @param tokens is the set of tokens parsed from the files.
+     * @param tokens is the list of tokens parsed from the files.
      * @param rootDirectory is the common rootDirectory of the files.
      * @return the string representation.
      */
-    public static String printTokens(TokenList tokens, File rootDirectory) {
+    public static String printTokens(List<Token> tokens, File rootDirectory) {
         return printTokens(tokens, rootDirectory, Optional.empty());
     }
 
     /**
      * Creates a string representation of a collection of files line by line and adds the tokens under the lines.
-     * @param tokenList is the set of tokens parsed from the files.
+     * @param tokenList is the list of tokens parsed from the files.
      * @param rootDirectory is the common directory of the files.
      * @param suffix is the optional view file suffix.
      * @return the string representation.
      */
-    public static String printTokens(TokenList tokenList, File rootDirectory, Optional<String> suffix) {
+    public static String printTokens(List<Token> tokenList, File rootDirectory, Optional<String> suffix) {
         PrinterOutputBuilder builder = new PrinterOutputBuilder();
         Map<String, List<Token>> fileToTokens = groupTokensByFile(tokenList);
 
@@ -124,7 +126,17 @@ public final class TokenPrinter {
         // Sort tokens by file and line -> tokens can be processed without any further checks
         List<String> lines = linesFromFile(file);
 
-        Map<Integer, List<Token>> lineNumbersToTokens = fileTokens.stream().collect(Collectors.groupingBy(Token::getLine));
+        int currentLine = Token.NO_VALUE;
+        Map<Integer, List<Token>> lineNumbersToTokens = new HashMap<>(fileTokens.size());
+        for (Token token : fileTokens) {
+            if (token.getLine() != Token.NO_VALUE) {
+                currentLine = token.getLine();
+            }
+            int line = token.getType() == TokenConstants.FILE_END ? lines.size() : currentLine;
+            List<Token> tokens = lineNumbersToTokens.containsKey(line) ? lineNumbersToTokens.get(line) : new ArrayList<>();
+            tokens.add(token);
+            lineNumbersToTokens.put(line, tokens);
+        }
 
         // create LineData for each line -- 1-based line index
         Stream<Integer> lineNumbers = PRINT_EMPTY_LINES ? IntStream.range(1, lines.size() + 1).boxed() : lineNumbersToTokens.keySet().stream();
@@ -132,8 +144,8 @@ public final class TokenPrinter {
                 .toList();
     }
 
-    private static Map<String, List<Token>> groupTokensByFile(TokenList tokenList) {
-        return tokenList.allTokens().stream().collect(Collectors.groupingBy(Token::getFile));
+    private static Map<String, List<Token>> groupTokensByFile(List<Token> tokens) {
+        return tokens.stream().collect(Collectors.groupingBy(Token::getFile));
     }
 
     /**

languages.api/src/test/java/de/jplag/TokenPrinterTest.java

@@ -2,9 +2,12 @@ package de.jplag;
 
 import static de.jplag.TokenConstants.FILE_END;
 import static de.jplag.simple.TestTokenConstants.STRING;
-import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
@@ -25,55 +28,55 @@ class TokenPrinterTest {
     void printMockDirectoriesAsSubmissions() {
 
         // See TokenPrinterTest.txt for the intended behaviour
-        TokenList tokens = new TokenList();
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 1, 1, "STRING".length()));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 2, 1, "STRING".length() + 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 3, 1, "STRING".length() + 2));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 4, 1, "STRING".length() + 10));
+        List<Token> tokens = new ArrayList<>();
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 1, 1, "STRING".length()));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 2, 1, "STRING".length() + 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 3, 1, "STRING".length() + 2));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 4, 1, "STRING".length() + 10));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 6, 3, 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 7, 9, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 6, 3, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 7, 9, 1));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 9, 1, 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 9, 10, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 9, 1, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 9, 10, 1));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 10, 1, 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 10, 5, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 10, 1, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 10, 5, 1));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 12, 1, 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 10, 5, 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 12, 10, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 12, 1, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 12, 5, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 12, 10, 1));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 14, 10, 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 14, 5, 1));
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 14, 1, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 14, 10, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 14, 5, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 14, 1, 1));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 16, -5, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 16, -5, 1));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 19, 100, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 19, 100, 1));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 22, 1, 100));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 22, 1, 100));
 
-        tokens.addToken(new TestToken(FILE_END, TEST_FILE_NAME, 24, 1, -1));
+        tokens.add(new TestToken(FILE_END, TEST_FILE_NAME, Token.NO_VALUE, Token.NO_VALUE, Token.NO_VALUE));
 
-        tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 100, 1, 1));
+        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 100, 1, 1));
 
         String output = TokenPrinter.printTokens(tokens, TEST_FILE_LOCATION.toFile());
-        logger.debug(output); // no additional newline required
+        logger.info(output); // no additional newline required
 
         testOutputCorrectness(TEST_FILE_NAME, tokens, output);
     }
 
-    private static void testOutputCorrectness(String fileName, TokenList tokens, String output) {
-        int lineIndex = 0;
+    private static void testOutputCorrectness(String fileName, List<Token> tokens, String output) {
+        int lineIndex = -1;
         int tokenIndex = 0;
-        boolean seenFileName = false;
         for (String line : output.lines().toList()) {
             if (line.isEmpty()) {
                 continue;
-            } else if (lineIndex == 0 && line.equals(fileName)) {
-                seenFileName = true;
-            } else if (line.startsWith("" + (lineIndex + 1))) {
+            } else if (lineIndex == -1) {
+                assertEquals(fileName, line);
+                lineIndex = 0;
+            } else if (line.startsWith(String.valueOf(lineIndex + 1))) {
                 lineIndex++;
             } else {
                 line = line.trim();
@@ -82,17 +85,15 @@ class TokenPrinterTest {
                     if (lineToken.isEmpty()) {
                         continue;
                     }
-                    Token currentToken = tokens.getToken(tokenIndex);
-                    if (lineToken.equalsIgnoreCase(currentToken.toString()) && currentToken.getLine() == lineIndex) {
-                        tokenIndex++;
-                    } else {
-                        fail("Expected token %s, but found %s".formatted(currentToken, lineToken));
+                    Token currentToken = tokens.get(tokenIndex);
+                    assertTrue(lineToken.equalsIgnoreCase(currentToken.toString()), "expected: %s, actual: %s".formatted(lineToken, currentToken));
+                    if (currentToken.getLine() != Token.NO_VALUE) {
+                        assertEquals(lineIndex, currentToken.getLine(), "invalid line for token " + currentToken);
                     }
+                    tokenIndex++;
                 }
             }
         }
-        if (!seenFileName) {
-            fail("Expected file name");
-        }
+        assertEquals(tokens.size() - 1, tokenIndex, "incorrect number of tokens printed");
     }
 }

languages.testutils/src/test/java/de/jplag/testutils/TokenUtils.java

@@ -3,7 +3,6 @@ package de.jplag.testutils;
 import java.util.List;
 
 import de.jplag.Token;
-import de.jplag.TokenList;
 
 public final class TokenUtils {
 
@@ -12,25 +11,23 @@ public final class TokenUtils {
     }
 
     /**
-     * Returns the type of all tokens in a {@link TokenList} that belong to a file.
-     * @param tokenList is the {@link TokenList}.
+     * Returns the type of all tokens that belong to a certain file.
+     * @param tokenList is the list of {@link Token Tokens}.
      * @param name is the name of the target file.
      * @return the immutable list of token types.
      */
-    public static List<Integer> tokenTypesByFile(TokenList tokenList, String name) {
-        var tokens = tokensByFile(tokenList, name);
-        return tokens.stream().map(Token::getType).toList();
+    public static List<Integer> tokenTypesByFile(List<Token> tokens, String name) {
+        return tokensByFile(tokens, name).stream().map(Token::getType).toList();
     }
 
     /**
-     * Returns the tokens in a {@link TokenList} that belong to a file.
-     * @param tokenList is the {@link TokenList}.
+     * Returns the tokens that belong to a certain file.
+     * @param tokenList is the list of {@link Token Tokens}.
      * @param name is the name of the target file.
      * @return the immutable list of tokens.
      */
-    public static List<Token> tokensByFile(TokenList tokenList, String name) {
-        var tokens = tokenList.allTokens().stream();
-        return tokens.filter(it -> it.getFile().startsWith(name)).toList();
+    public static List<Token> tokensByFile(List<Token> tokens, String name) {
+        return tokens.stream().filter(it -> it.getFile().startsWith(name)).toList();
     }
 
 }

languages/cpp/src/main/java/de/jplag/cpp/Language.java

@@ -1,10 +1,11 @@
 package de.jplag.cpp;
 
 import java.io.File;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 @MetaInfServices(de.jplag.Language.class)
 public class Language implements de.jplag.Language {
@@ -37,7 +38,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File dir, String[] files) {
+    public List<Token> parse(File dir, String[] files) {
         return this.scanner.scan(dir, files);
     }
 

languages/cpp/src/main/java/de/jplag/cpp/Scanner.java

@@ -1,14 +1,16 @@
 package de.jplag.cpp;
 
 import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenList;
+import de.jplag.TokenConstants;
 
 public class Scanner extends AbstractParser {
     private String currentFile;
 
-    private TokenList tokens;
+    private List<de.jplag.Token> tokens;
 
     /**
      * Creates the parser.
@@ -17,8 +19,8 @@ public class Scanner extends AbstractParser {
         super();
     }
 
-    public TokenList scan(File directory, String[] files) {
-        tokens = new TokenList();
+    public List<de.jplag.Token> scan(File directory, String[] files) {
+        tokens = new ArrayList<>();
         errors = 0;
         for (String currentFile : files) {
             this.currentFile = currentFile;
@@ -26,13 +28,13 @@ public class Scanner extends AbstractParser {
             if (!CPPScanner.scanFile(directory, currentFile, this)) {
                 errors++;
             }
-            tokens.addToken(new CPPToken(CPPTokenConstants.FILE_END, currentFile));
+            tokens.add(new CPPToken(TokenConstants.FILE_END, currentFile));
         }
         return tokens;
     }
 
     public void add(int type, Token token) {
         int length = token.endColumn - token.beginColumn + 1;
-        tokens.addToken(new CPPToken(type, currentFile, token.beginLine, token.beginColumn, length));
+        tokens.add(new CPPToken(type, currentFile, token.beginLine, token.beginColumn, length));
     }
 }

languages/csharp-6/src/main/java/de/jplag/csharp/CSharpParserAdapter.java

@@ -3,6 +3,7 @@ package de.jplag.csharp;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.antlr.v4.runtime.CharStreams;
@@ -12,7 +13,8 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenList;
+import de.jplag.Token;
+import de.jplag.TokenConstants;
 import de.jplag.csharp.grammar.CSharpLexer;
 import de.jplag.csharp.grammar.CSharpParser;
 
@@ -22,7 +24,7 @@ import de.jplag.csharp.grammar.CSharpParser;
  * @author Timur Saglam
  */
 public class CSharpParserAdapter extends AbstractParser {
-    private TokenList tokens;
+    private List<Token> tokens;
     private String currentFile;
 
     /**
@@ -38,14 +40,14 @@ public class CSharpParserAdapter extends AbstractParser {
      * @param fileNames is the list of file names.
      * @return the list of parsed tokens.
      */
-    public TokenList parse(File directory, List<String> fileNames) {
-        tokens = new TokenList();
+    public List<Token> parse(File directory, List<String> fileNames) {
+        tokens = new ArrayList<>();
         errors = 0;
         for (String fileName : fileNames) {
             if (!parseFile(directory, fileName)) {
                 errors++;
             }
-            tokens.addToken(new CSharpToken(CSharpTokenConstants.FILE_END, fileName, -1, -1, -1));
+            tokens.add(new CSharpToken(TokenConstants.FILE_END, fileName, -1, -1, -1));
         }
         return tokens;
     }
@@ -77,6 +79,6 @@ public class CSharpParserAdapter extends AbstractParser {
     }
 
     /* package-private */ void addToken(int type, int line, int column, int length) {
-        tokens.addToken(new CSharpToken(type, currentFile, line, column, length));
+        tokens.add(new CSharpToken(type, currentFile, line, column, length));
     }
 }

languages/csharp-6/src/main/java/de/jplag/csharp/Language.java

@@ -2,10 +2,11 @@ package de.jplag.csharp;
 
 import java.io.File;
 import java.util.Arrays;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 /**
  * C# language with full support of C# 6 features and below.
@@ -45,7 +46,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File dir, String[] files) {
+    public List<Token> parse(File dir, String[] files) {
         return parser.parse(dir, Arrays.asList(files));
     }
 

languages/csharp-6/src/test/java/de/jplag/csharp/MinimalCSharpFrontendTest.java

@@ -16,7 +16,6 @@ import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.TokenPrinter;
 
 class MinimalCSharpFrontendTest {
@@ -44,12 +43,12 @@ class MinimalCSharpFrontendTest {
 
         // Parse test input
         String[] input = new String[] {TEST_SUBJECT};
-        TokenList result = frontend.parse(baseDirectory, input);
+        List<Token> result = frontend.parse(baseDirectory, input);
         logger.info(TokenPrinter.printTokens(result, baseDirectory));
 
         // Compare parsed tokens:
         assertEquals(expectedToken.size(), result.size());
-        List<Integer> actualToken = result.allTokens().stream().map(Token::getType).collect(toList());
+        List<Integer> actualToken = result.stream().map(Token::getType).collect(toList());
         assertEquals(expectedToken, actualToken);
     }
 

languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/parser/DynamicEcoreParser.java

@@ -23,7 +23,7 @@ public class DynamicEcoreParser extends EcoreParser {
     @Override
     public void addToken(int type, EObject source) {
         MetamodelToken token = new DynamicMetamodelToken(type, currentFile, source);
-        treeView.addToken(token, visitor.getCurrentTreeDepth(), NO_PREFIX);
-        tokens.addToken(token);
+        MetamodelToken metadataEnrichedToken = treeView.convertToMetadataEnrichedTokenAndAdd(token, visitor.getCurrentTreeDepth(), NO_PREFIX);
+        tokens.add(metadataEnrichedToken);
     }
 }

languages/emf-metamodel-dynamic/src/test/java/de/jplag/emf/dynamic/MinimalDynamicMetamodelTest.java

@@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.File;
 import java.nio.file.Path;
+import java.util.List;
 import java.util.Optional;
 
 import org.junit.jupiter.api.AfterEach;
@@ -14,7 +15,7 @@ import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 import de.jplag.TokenPrinter;
 import de.jplag.testutils.FileUtil;
 import de.jplag.testutils.TokenUtils;
@@ -37,10 +38,10 @@ class MinimalDynamicMetamodelTest {
 
     @Test
     void testBookstoreMetamodels() {
-        TokenList result = frontend.parse(baseDirectory, TEST_SUBJECTS);
+        List<Token> result = frontend.parse(baseDirectory, TEST_SUBJECTS);
         logger.debug(TokenPrinter.printTokens(result, baseDirectory, Optional.of(Language.VIEW_FILE_SUFFIX)));
         logger.info(("Dynamic token set: " + DynamicMetamodelTokenConstants.getTokenStrings()));
-        logger.info("parsed tokens: " + result.allTokens().toString());
+        logger.info("parsed tokens: " + result.toString());
         assertEquals(7, DynamicMetamodelTokenConstants.getTokenStrings().size());
         assertEquals(64, result.size());
 

languages/emf-metamodel/src/main/java/de/jplag/emf/Language.java

@@ -2,11 +2,12 @@ package de.jplag.emf;
 
 import java.io.File;
 import java.util.Arrays;
+import java.util.List;
 
 import org.eclipse.emf.ecore.EcorePackage;
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 import de.jplag.emf.parser.EcoreParser;
 
 /**
@@ -53,7 +54,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File dir, String[] files) {
+    public List<Token> parse(File dir, String[] files) {
         return parser.parse(dir, Arrays.asList(files));
     }
 

languages/emf-metamodel/src/main/java/de/jplag/emf/MetamodelToken.java

@@ -36,6 +36,20 @@ public class MetamodelToken extends Token implements MetamodelTokenConstants {
         this.eObject = Optional.empty();
     }
 
+    /**
+     * Creates a token with column and length information.
+     * @param type is the token type.
+     * @param file is the name of the source code file.
+     * @param line is the line index in the source code where the token resides. Cannot be smaller than 1.
+     * @param column is the column index, meaning where the token starts in the line.
+     * @param length is the length of the token in the source code.
+     * @param eObject is the corresponding eObject in the model from which this token was extracted
+     */
+    public MetamodelToken(int type, String file, int line, int column, int length, Optional<EObject> eObject) {
+        super(type, file, line, column, length);
+        this.eObject = eObject;
+    }
+
     /**
      * @return the optional corresponding EObject of the token.
      */

languages/emf-metamodel/src/main/java/de/jplag/emf/parser/EcoreParser.java

@@ -1,13 +1,14 @@
 package de.jplag.emf.parser;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.eclipse.emf.ecore.EObject;
 
 import de.jplag.AbstractParser;
+import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.emf.Language;
 import de.jplag.emf.MetamodelToken;
 import de.jplag.emf.util.AbstractMetamodelVisitor;
@@ -19,7 +20,7 @@ import de.jplag.emf.util.MetamodelTreeView;
  * @author Timur Saglam
  */
 public class EcoreParser extends AbstractParser {
-    protected TokenList tokens;
+    protected List<Token> tokens;
     protected String currentFile;
     protected MetamodelTreeView treeView;
     protected AbstractMetamodelVisitor visitor;
@@ -37,9 +38,9 @@ public class EcoreParser extends AbstractParser {
      * @param fileNames is the list of file names.
      * @return the list of parsed tokens.
      */
-    public TokenList parse(File directory, List<String> fileNames) {
+    public List<Token> parse(File directory, List<String> fileNames) {
         errors = 0;
-        tokens = new TokenList();
+        tokens = new ArrayList<>();
         for (String fileName : fileNames) {
             currentFile = fileName;
             String filePath = fileName.isEmpty() ? directory.toString() : directory.toString() + File.separator + fileName;
@@ -62,7 +63,7 @@ public class EcoreParser extends AbstractParser {
                 visitor = createMetamodelVisitor();
                 visitor.visit(root);
             }
-            tokens.addToken(new MetamodelToken(TokenConstants.FILE_END, currentFile));
+            tokens.add(new MetamodelToken(TokenConstants.FILE_END, currentFile));
             treeView.writeToFile(Language.VIEW_FILE_SUFFIX);
         }
     }
@@ -77,8 +78,8 @@ public class EcoreParser extends AbstractParser {
 
     public void addToken(int type, EObject source, String prefix) {
         MetamodelToken token = new MetamodelToken(type, currentFile, source);
-        treeView.addToken(token, visitor.getCurrentTreeDepth(), prefix);
-        tokens.addToken(token);
+        MetamodelToken metadataEnrichedToken = treeView.convertToMetadataEnrichedTokenAndAdd(token, visitor.getCurrentTreeDepth(), prefix);
+        tokens.add(metadataEnrichedToken);
     }
 
     public void addToken(int type, EObject source) {

languages/emf-metamodel/src/main/java/de/jplag/emf/util/MetamodelTreeView.java

@@ -4,11 +4,14 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.util.Optional;
 
 import org.eclipse.emf.ecore.ENamedElement;
+import org.eclipse.emf.ecore.EObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import de.jplag.Token;
 import de.jplag.emf.MetamodelToken;
 
 /**
@@ -34,12 +37,18 @@ public class MetamodelTreeView {
     }
 
     /**
-     * Adds a token to the view, thus adding the index information to the token.
+     * Adds a token to the view, thus adding the index information to the token. Returns a new token enriched with the index
+     * metadata.
      * @param token is the token to add.
      * @param treeDepth is the current containment tree depth, required for the indentation.
      */
-    public void addToken(MetamodelToken token, int treeDepth, String prefix) {
-        token.getEObject().ifPresent(it -> {
+    public MetamodelToken convertToMetadataEnrichedTokenAndAdd(MetamodelToken token, int treeDepth, String prefix) {
+        int length = Token.NO_VALUE;
+        int line = Token.NO_VALUE;
+        int column = Token.NO_VALUE;
+        Optional<EObject> optionalEObject = token.getEObject();
+        if (optionalEObject.isPresent()) {
+            EObject eObject = optionalEObject.get();
             if (prefix.isEmpty() && treeDepth > 0) {
                 lineIndex++;
                 columnIndex = 0;
@@ -47,10 +56,10 @@ public class MetamodelTreeView {
             }
 
             String tokenText = token.toString();
-            if (it instanceof ENamedElement element) {
+            if (eObject instanceof ENamedElement element) {
                 tokenText = element.getName() + " : " + tokenText;
             }
-            token.setLength(tokenText.length());
+            length = tokenText.length();
 
             if (prefix.isEmpty()) {
                 for (int i = 0; i < treeDepth; i++) {
@@ -63,12 +72,12 @@ public class MetamodelTreeView {
                 columnIndex += prefix.length();
             }
 
-            token.setLine(lineIndex + 1);
-            token.setColumn(columnIndex + 1);
+            line = lineIndex + 1;
+            column = columnIndex + 1;
 
             columnIndex += tokenText.length();
-
-        });
+        }
+        return new MetamodelToken(token.getType(), token.getFile(), line, column, length, token.getEObject());
     }
 
     /**

languages/emf-metamodel/src/test/java/de/jplag/emf/MinimalMetamodelTest.java

@@ -6,6 +6,7 @@ import java.io.File;
 import java.lang.reflect.Field;
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.List;
 import java.util.Optional;
 
 import org.junit.jupiter.api.AfterEach;
@@ -14,7 +15,7 @@ import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 import de.jplag.TokenPrinter;
 import de.jplag.testutils.FileUtil;
 import de.jplag.testutils.TokenUtils;
@@ -37,13 +38,13 @@ class MinimalMetamodelTest {
 
     @Test
     void testBookstoreMetamodels() {
-        TokenList result = frontend.parse(baseDirectory, TEST_SUBJECTS);
+        List<Token> result = frontend.parse(baseDirectory, TEST_SUBJECTS);
 
         logger.debug(TokenPrinter.printTokens(result, baseDirectory, Optional.of(Language.VIEW_FILE_SUFFIX)));
         Field[] fields = MetamodelTokenConstants.class.getFields();
         var constants = Arrays.stream(fields).map(Field::getName).filter(it -> !it.equals("NUM_DIFF_TOKENS")).toList();
         logger.info(("Handcrafted token set: " + constants));
-        logger.info("Parsed tokens: " + result.allTokens().toString());
+        logger.info("Parsed tokens: " + result.toString());
         assertEquals(21, constants.size());
         assertEquals(43, result.size());
 

languages/golang/src/main/java/de/jplag/golang/GoParserAdapter.java

@@ -3,6 +3,8 @@ package de.jplag.golang;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
@@ -11,16 +13,16 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenList;
+import de.jplag.Token;
 import de.jplag.golang.grammar.GoLexer;
 import de.jplag.golang.grammar.GoParser;
 
 public class GoParserAdapter extends AbstractParser {
     private String currentFile;
-    private TokenList tokens;
+    private List<Token> tokens;
 
-    public TokenList parse(File directory, String[] fileNames) {
-        tokens = new TokenList();
+    public List<Token> parse(File directory, String[] fileNames) {
+        tokens = new ArrayList<>();
         for (String file : fileNames) {
             if (!parseFile(directory, file)) {
                 errors++;
@@ -54,6 +56,6 @@ public class GoParserAdapter extends AbstractParser {
     }
 
     public void addToken(int tokenType, int line, int column, int length) {
-        tokens.addToken(new GoToken(tokenType, currentFile, line, column, length));
+        tokens.add(new GoToken(tokenType, currentFile, line, column, length));
     }
 }

languages/golang/src/main/java/de/jplag/golang/Language.java

@@ -1,10 +1,11 @@
 package de.jplag.golang;
 
 import java.io.File;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 @MetaInfServices(de.jplag.Language.class)
 public class Language implements de.jplag.Language {
@@ -40,7 +41,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File directory, String[] files) {
+    public List<Token> parse(File directory, String[] files) {
         return parserAdapter.parse(directory, files);
     }
 

languages/golang/src/test/java/de/jplag/golang/GoFrontendTest.java

@@ -21,7 +21,6 @@ import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.TokenPrinter;
 
 class GoFrontendTest {
@@ -58,7 +57,7 @@ class GoFrontendTest {
     @Test
     void parseTestFiles() {
         for (String fileName : testFiles) {
-            TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
+            List<Token> tokens = language.parse(testFileLocation, new String[] {fileName});
             String output = TokenPrinter.printTokens(tokens, testFileLocation);
             logger.info(output);
 
@@ -87,9 +86,9 @@ class GoFrontendTest {
     /**
      * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
      * @param fileName a code sample file name
-     * @param tokens the TokenList generated from the sample
+     * @param tokens the list of tokens generated from the sample
      */
-    private void testSourceCoverage(String fileName, TokenList tokens) {
+    private void testSourceCoverage(String fileName, List<Token> tokens) {
         File testFile = new File(testFileLocation, fileName);
 
         List<String> lines = null;
@@ -103,7 +102,7 @@ class GoFrontendTest {
         // All lines that contain code
         var codeLines = getCodeLines(lines);
         // All lines that contain a token
-        var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().boxed().toList();
+        var tokenLines = tokens.stream().map(Token::getLine).distinct().toList();
 
         if (codeLines.size() > tokenLines.size()) {
             List<Integer> missedLinesIndices = new ArrayList<>(codeLines);
@@ -153,11 +152,11 @@ class GoFrontendTest {
 
     /**
      * Confirms that all Token types are 'reachable' with a complete code example.
-     * @param tokens TokenList which is supposed to contain all types of tokens
+     * @param tokens list of tokens which is supposed to contain all types of tokens
      * @param fileName The file name of the complete code example
      */
-    private void testTokenCoverage(TokenList tokens, String fileName) {
-        var foundTokens = tokens.allTokens().stream().parallel().mapToInt(Token::getType).sorted().distinct().boxed().toList();
+    private void testTokenCoverage(List<Token> tokens, String fileName) {
+        var foundTokens = tokens.stream().parallel().map(Token::getType).sorted().distinct().toList();
 
         // Exclude SEPARATOR_TOKEN, as it does not occur
         var missingTokenTypes = IntStream.range(0, GoTokenConstants.NUM_DIFF_TOKENS).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).boxed()

languages/java/src/main/java/de/jplag/java/Language.java

@@ -1,10 +1,11 @@
 package de.jplag.java;
 
 import java.io.File;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 /**
  * Language for Java 9 and newer.
@@ -40,7 +41,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File directory, String[] files) {
+    public List<Token> parse(File directory, String[] files) {
         return this.parser.parse(directory, files);
     }
 

languages/java/src/main/java/de/jplag/java/Parser.java

@@ -1,13 +1,15 @@
 package de.jplag.java;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 public class Parser extends AbstractParser {
-    private TokenList tokens;
+    private List<Token> tokens;
 
     /**
      * Creates the parser.
@@ -16,8 +18,8 @@ public class Parser extends AbstractParser {
         super();
     }
 
-    public TokenList parse(File directory, String[] files) {
-        tokens = new TokenList();
+    public List<Token> parse(File directory, String[] files) {
+        tokens = new ArrayList<>();
         errors = 0;
         var pathedFiles = Arrays.stream(files).map(it -> new File(directory, it)).toList();
         errors += new JavacAdapter().parseFiles(directory, pathedFiles, this);
@@ -25,7 +27,7 @@ public class Parser extends AbstractParser {
     }
 
     public void add(int type, String filename, long line, long column, long length) {
-        tokens.addToken(new JavaToken(type, filename, (int) line, (int) column, (int) length));
+        tokens.add(new JavaToken(type, filename, (int) line, (int) column, (int) length));
     }
 
     public void increaseErrors() {

languages/kotlin/src/main/java/de/jplag/kotlin/KotlinParserAdapter.java

@@ -3,6 +3,8 @@ package de.jplag.kotlin;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
@@ -11,8 +13,8 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
+import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.kotlin.grammar.KotlinLexer;
 import de.jplag.kotlin.grammar.KotlinParser;
 
@@ -20,7 +22,7 @@ public class KotlinParserAdapter extends AbstractParser {
 
     public static final int NOT_SET = -1;
     private String currentFile;
-    private TokenList tokens;
+    private List<Token> tokens;
 
     /**
      * Creates the KotlinParserAdapter
@@ -30,18 +32,18 @@ public class KotlinParserAdapter extends AbstractParser {
     }
 
     /**
-     * Parsers a list of files into a single {@link TokenList}.
+     * Parsers a list of files into a single list of {@link Token}s.
      * @param directory the directory of the files.
      * @param fileNames the file names of the files.
-     * @return a {@link TokenList} containing all tokens of all files.
+     * @return a list containing all tokens of all files.
      */
-    public TokenList parse(File directory, String[] fileNames) {
-        tokens = new TokenList();
+    public List<Token> parse(File directory, String[] fileNames) {
+        tokens = new ArrayList<>();
         for (String file : fileNames) {
             if (!parseFile(directory, file)) {
                 errors++;
             }
-            tokens.addToken(new KotlinToken(TokenConstants.FILE_END, file, NOT_SET, NOT_SET, NOT_SET));
+            tokens.add(new KotlinToken(TokenConstants.FILE_END, file, NOT_SET, NOT_SET, NOT_SET));
         }
         return tokens;
     }
@@ -71,13 +73,13 @@ public class KotlinParserAdapter extends AbstractParser {
     }
 
     /**
-     * Adds a new {@link de.jplag.Token} to the current {@link TokenList}.
-     * @param tokenType the type of the new {@link de.jplag.Token}
+     * Adds a new {@link Token} to the current token list.
+     * @param tokenType the type of the new {@link Token}
      * @param line the line of the Token in the current file
      * @param column the start column of the Token in the line
      * @param length the length of the Token
      */
     /* package-private */ void addToken(int tokenType, int line, int column, int length) {
-        tokens.addToken(new KotlinToken(tokenType, currentFile, line, column, length));
+        tokens.add(new KotlinToken(tokenType, currentFile, line, column, length));
     }
 }

languages/kotlin/src/main/java/de/jplag/kotlin/Language.java

@@ -1,10 +1,11 @@
 package de.jplag.kotlin;
 
 import java.io.File;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 /**
  * This represents the Kotlin language as a language supported by JPlag.
@@ -43,7 +44,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File directory, String[] files) {
+    public List<Token> parse(File directory, String[] files) {
         return parserAdapter.parse(directory, files);
     }
 

languages/kotlin/src/test/java/de/jplag/kotlin/KotlinFrontendTest.java

@@ -18,7 +18,6 @@ import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.TokenPrinter;
 
 class KotlinFrontendTest {
@@ -58,7 +57,7 @@ class KotlinFrontendTest {
     @Test
     void parseTestFiles() {
         for (String fileName : testFiles) {
-            TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
+            List<Token> tokens = language.parse(testFileLocation, new String[] {fileName});
             String output = TokenPrinter.printTokens(tokens, testFileLocation);
             logger.info(output);
 
@@ -89,9 +88,9 @@ class KotlinFrontendTest {
     /**
      * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
      * @param fileName a code sample file name
-     * @param tokens the TokenList generated from the sample
+     * @param tokens the list of tokens generated from the sample
      */
-    private void testSourceCoverage(String fileName, TokenList tokens) {
+    private void testSourceCoverage(String fileName, List<Token> tokens) {
         File testFile = new File(testFileLocation, fileName);
 
         try {
@@ -100,7 +99,7 @@ class KotlinFrontendTest {
             // All lines that contain code
             var codeLines = getCodeLines(lines);
             // All lines that contain token
-            var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().toArray();
+            var tokenLines = tokens.stream().mapToInt(Token::getLine).filter(line -> line != Token.NO_VALUE).distinct().toArray();
 
             if (codeLines.length > tokenLines.length) {
                 var diffLine = IntStream.range(0, codeLines.length)
@@ -149,11 +148,11 @@ class KotlinFrontendTest {
 
     /**
      * Confirms that all Token types are 'reachable' with a complete code example.
-     * @param tokens TokenList which is supposed to contain all types of tokens
+     * @param tokens list of tokens which is supposed to contain all types of tokens
      * @param fileName The file name of the complete code example
      */
-    private void testTokenCoverage(TokenList tokens, String fileName) {
-        var foundTokens = tokens.allTokens().stream().parallel().mapToInt(Token::getType).sorted().distinct().toArray();
+    private void testTokenCoverage(List<Token> tokens, String fileName) {
+        var foundTokens = tokens.stream().parallel().mapToInt(Token::getType).sorted().distinct().toArray();
         // Exclude SEPARATOR_TOKEN, as it does not occur
         var allTokens = IntStream.range(0, KotlinTokenConstants.NUMBER_DIFF_TOKENS).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).toArray();
 

languages/python-3/src/main/java/de/jplag/python3/Language.java

@@ -1,10 +1,11 @@
 package de.jplag.python3;
 
 import java.io.File;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 @MetaInfServices(de.jplag.Language.class)
 public class Language implements de.jplag.Language {
@@ -38,7 +39,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File dir, String[] files) {
+    public List<Token> parse(File dir, String[] files) {
         return this.parser.parse(dir, files);
     }
 

languages/python-3/src/main/java/de/jplag/python3/Parser.java

@@ -4,6 +4,8 @@ import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CharStreams;
@@ -13,14 +15,14 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenList;
+import de.jplag.TokenConstants;
 import de.jplag.python3.grammar.Python3Lexer;
 import de.jplag.python3.grammar.Python3Parser;
 import de.jplag.python3.grammar.Python3Parser.File_inputContext;
 
 public class Parser extends AbstractParser {
 
-    private TokenList tokens = new TokenList();
+    private List<de.jplag.Token> tokens;
     private String currentFile;
 
     /**
@@ -30,15 +32,15 @@ public class Parser extends AbstractParser {
         super();
     }
 
-    public TokenList parse(File directory, String[] files) {
-        tokens = new TokenList();
+    public List<de.jplag.Token> parse(File directory, String[] files) {
+        tokens = new ArrayList<>();
         errors = 0;
         for (String file : files) {
             logger.trace("Parsing file {}", file);
             if (!parseFile(directory, file)) {
                 errors++;
             }
-            tokens.addToken(new Python3Token(Python3TokenConstants.FILE_END, file, -1, -1, -1));
+            tokens.add(new Python3Token(TokenConstants.FILE_END, file, -1, -1, -1));
         }
         return tokens;
     }
@@ -77,12 +79,12 @@ public class Parser extends AbstractParser {
     }
 
     public void add(int type, Token token) {
-        tokens.addToken(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(), token.getCharPositionInLine() + 1,
+        tokens.add(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(), token.getCharPositionInLine() + 1,
                 token.getText().length()));
     }
 
     public void addEnd(int type, Token token) {
-        tokens.addToken(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(),
-                tokens.getToken(tokens.size() - 1).getColumn() + 1, 0));
+        tokens.add(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(),
+                tokens.get(tokens.size() - 1).getColumn() + 1, 0));
     }
 }

languages/rlang/src/main/java/de/jplag/rlang/Language.java

@@ -1,10 +1,11 @@
 package de.jplag.rlang;
 
 import java.io.File;
+import java.util.List;
 
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 /**
  * This represents the R language as a language supported by JPlag.
@@ -43,7 +44,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File directory, String[] files) {
+    public List<Token> parse(File directory, String[] files) {
         return parserAdapter.parse(directory, files);
     }
 

languages/rlang/src/main/java/de/jplag/rlang/RParserAdapter.java

@@ -3,6 +3,8 @@ package de.jplag.rlang;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
@@ -11,7 +13,7 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenList;
+import de.jplag.Token;
 import de.jplag.rlang.grammar.RFilter;
 import de.jplag.rlang.grammar.RLexer;
 import de.jplag.rlang.grammar.RParser;
@@ -23,7 +25,7 @@ import de.jplag.rlang.grammar.RParser;
 public class RParserAdapter extends AbstractParser implements RTokenConstants {
 
     private String currentFile;
-    private TokenList tokens;
+    private List<Token> tokens;
 
     /**
      * Creates the RParserAdapter
@@ -33,19 +35,19 @@ public class RParserAdapter extends AbstractParser implements RTokenConstants {
     }
 
     /**
-     * Parsers a list of files into a single {@link TokenList}.
+     * Parsers a list of files into a single token list of {@link Token}s.
      * @param directory the directory of the files.
      * @param fileNames the file names of the files.
-     * @return a {@link TokenList} containing all tokens of all files.
+     * @return a list containing all tokens of all files.
      */
-    public TokenList parse(File directory, String[] fileNames) {
-        tokens = new TokenList();
+    public List<Token> parse(File directory, String[] fileNames) {
+        tokens = new ArrayList<>();
         errors = 0;
         for (String fileName : fileNames) {
             if (!parseFile(directory, fileName)) {
                 errors++;
             }
-            tokens.addToken(new RToken(FILE_END, fileName, -1, -1, -1));
+            tokens.add(new RToken(FILE_END, fileName, -1, -1, -1));
         }
         return tokens;
     }
@@ -82,14 +84,14 @@ public class RParserAdapter extends AbstractParser implements RTokenConstants {
     }
 
     /**
-     * Adds a new {@link de.jplag.Token} to the current {@link TokenList}.
-     * @param type the type of the new {@link de.jplag.Token}
+     * Adds a new {@link Token} to the current token list.
+     * @param type the type of the new {@link Token}
      * @param line the line of the Token in the current file
      * @param start the start column of the Token in the line
      * @param length the length of the Token
      */
     /* package-private */ void addToken(int type, int line, int start, int length) {
-        tokens.addToken(new RToken(type, currentFile, line, start, length));
+        tokens.add(new RToken(type, currentFile, line, start, length));
 
     }
 }

languages/rlang/src/test/java/de/jplag/rlang/RFrontendTest.java

@@ -17,7 +17,6 @@ import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.TokenPrinter;
 
 class RFrontendTest {
@@ -46,7 +45,7 @@ class RFrontendTest {
     @Test
     void parseTestFiles() {
         for (String fileName : testFiles) {
-            TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
+            List<Token> tokens = language.parse(testFileLocation, new String[] {fileName});
             String output = TokenPrinter.printTokens(tokens, testFileLocation);
             logger.info(output);
 
@@ -59,9 +58,9 @@ class RFrontendTest {
     /**
      * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
      * @param fileName a code sample file name
-     * @param tokens the TokenList generated from the sample
+     * @param tokens the list of tokens generated from the sample
      */
-    private void testSourceCoverage(String fileName, TokenList tokens) {
+    private void testSourceCoverage(String fileName, List<Token> tokens) {
         File testFile = new File(testFileLocation, fileName);
 
         try {
@@ -71,7 +70,7 @@ class RFrontendTest {
             // All lines that contain code
             var codeLines = IntStream.range(1, lines.size() + 1).filter(idx -> !lines.get(idx - 1).matches(emptyLineExpression)).toArray();
             // All lines that contain token
-            var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().toArray();
+            var tokenLines = tokens.stream().mapToInt(Token::getLine).filter(line -> line != Token.NO_VALUE).distinct().toArray();
 
             if (codeLines.length > tokenLines.length) {
                 var diffLine = IntStream.range(0, codeLines.length)
@@ -88,11 +87,11 @@ class RFrontendTest {
 
     /**
      * Confirms that all Token types are 'reachable' with a complete code example.
-     * @param tokens TokenList which is supposed to contain all types of tokens
+     * @param tokens list of tokens which is supposed to contain all types of tokens
      * @param fileName The file name of the complete code example
      */
-    private void testTokenCoverage(TokenList tokens, String fileName) {
-        var foundTokens = tokens.allTokens().stream().parallel().mapToInt(Token::getType).sorted().distinct().toArray();
+    private void testTokenCoverage(List<Token> tokens, String fileName) {
+        var foundTokens = tokens.stream().parallel().mapToInt(Token::getType).sorted().distinct().toArray();
         // Exclude SEPARATOR_TOKEN, as it does not occur
         var allTokens = IntStream.range(0, RTokenConstants.NUM_DIFF_TOKENS).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).toArray();
 

languages/rust/src/main/java/de/jplag/rust/Language.java

@@ -1,8 +1,9 @@
 package de.jplag.rust;
 
 import java.io.File;
+import java.util.List;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 public class Language implements de.jplag.Language {
 
@@ -38,7 +39,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File directory, String[] files) {
+    public List<Token> parse(File directory, String[] files) {
         return parserAdapter.parse(directory, files);
     }
 

languages/rust/src/main/java/de/jplag/rust/RustParserAdapter.java

@@ -3,6 +3,8 @@ package de.jplag.rust;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
@@ -11,8 +13,8 @@ import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
+import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.rust.grammar.RustLexer;
 import de.jplag.rust.grammar.RustParser;
 
@@ -20,22 +22,22 @@ public class RustParserAdapter extends AbstractParser {
 
     private static final int NOT_SET = -1;
     private String currentFile;
-    private TokenList tokens;
+    private List<Token> tokens;
 
     /**
-     * Parsers a list of files into a single {@link TokenList}.
+     * Parsers a list of files into a single list of {@link Token}s.
      * @param directory the directory of the files.
      * @param fileNames the file names of the files.
-     * @return a {@link TokenList} containing all tokens of all files.
+     * @return a list containing all tokens of all files.
      */
-    public TokenList parse(File directory, String[] fileNames) {
-        tokens = new TokenList();
+    public List<Token> parse(File directory, String[] fileNames) {
+        tokens = new ArrayList<>();
         errors = 0;
         for (String fileName : fileNames) {
             if (!parseFile(directory, fileName)) {
                 errors++;
             }
-            tokens.addToken(new RustToken(TokenConstants.FILE_END, fileName, NOT_SET, NOT_SET, NOT_SET));
+            tokens.add(new RustToken(TokenConstants.FILE_END, fileName, NOT_SET, NOT_SET, NOT_SET));
         }
         return tokens;
     }
@@ -68,14 +70,14 @@ public class RustParserAdapter extends AbstractParser {
     }
 
     /**
-     * Adds a new {@link de.jplag.Token} to the current {@link TokenList}.
-     * @param type the type of the new {@link de.jplag.Token}
+     * Adds a new {@link Token} to the current token list.
+     * @param type the type of the new {@link Token}
      * @param line the line of the Token in the current file
      * @param start the start column of the Token in the line
      * @param length the length of the Token
      */
     /* package-private */ void addToken(int type, int line, int start, int length) {
-        tokens.addToken(new RustToken(type, currentFile, line, start, length));
+        tokens.add(new RustToken(type, currentFile, line, start, length));
 
     }
 }

languages/rust/src/test/java/de/jplag/rust/RustFrontendTest.java

@@ -1,6 +1,6 @@
 package de.jplag.rust;
 
-import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.File;
 import java.io.IOException;
@@ -17,7 +17,6 @@ import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
 import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.TokenPrinter;
 
 class RustFrontendTest {
@@ -37,6 +36,7 @@ class RustFrontendTest {
     private static final String EMPTY_STRING = "";
     private static final String RUST_SHEBANG = "#!.*$";
     private static final double EPSILON = 1E-6;
+    public static final double BASELINE_COVERAGE = 0.75;
 
     private final Logger logger = LoggerFactory.getLogger("Rust frontend test");
     private final String[] testFiles = new String[] {"deno_core_runtime.rs", COMPLETE_TEST_FILE};
@@ -51,7 +51,7 @@ class RustFrontendTest {
     @Test
     void parseTestFiles() {
         for (String fileName : testFiles) {
-            TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
+            List<Token> tokens = language.parse(testFileLocation, new String[] {fileName});
             String output = TokenPrinter.printTokens(tokens, testFileLocation);
             logger.info(output);
 
@@ -64,9 +64,9 @@ class RustFrontendTest {
     /**
      * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
      * @param fileName a code sample file name
-     * @param tokens the TokenList generated from the sample
+     * @param tokens the list of tokens generated from the sample
      */
-    private void testSourceCoverage(String fileName, TokenList tokens) {
+    private void testSourceCoverage(String fileName, List<Token> tokens) {
         File testFile = new File(testFileLocation, fileName);
 
         try {
@@ -75,7 +75,7 @@ class RustFrontendTest {
             // All lines that contain code
             var codeLines = new ArrayList<>(getCodeLines(lines));
             // All lines that contain token
-            var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().boxed().toList();
+            var tokenLines = tokens.stream().map(Token::getLine).filter(line -> line != Token.NO_VALUE).distinct().toList();
 
             // Keep only lines that have no tokens
             codeLines.removeAll(tokenLines);
@@ -86,15 +86,12 @@ class RustFrontendTest {
             } else {
                 logger.info("Coverage: %.1f%%.".formatted(coverage * 100));
                 logger.info("Missing lines {}", codeLines);
-                if (coverage - 0.9 <= EPSILON) {
-                    // TODO use fail() instead when frontend is ready
-                    logger.error("Source coverage is unsatisfactory");
-                }
+                assertTrue(coverage - BASELINE_COVERAGE >= EPSILON, "Source coverage is unsatisfactory");
             }
 
         } catch (IOException exception) {
             logger.info("Error while reading test file %s".formatted(fileName), exception);
-            fail();
+            assertTrue(false);
         }
     }
 
@@ -124,11 +121,11 @@ class RustFrontendTest {
 
     /**
      * Confirms that all Token types are 'reachable' with a complete code example.
-     * @param tokens TokenList which is supposed to contain all types of tokens
+     * @param tokens list of tokens which is supposed to contain all types of tokens
      * @param fileName The file name of the complete code example
      */
-    private void testTokenCoverage(TokenList tokens, String fileName) {
-        var foundTokens = tokens.allTokens().stream().mapToInt(Token::getType).distinct().boxed().toList();
+    private void testTokenCoverage(List<Token> tokens, String fileName) {
+        var foundTokens = tokens.stream().map(Token::getType).distinct().toList();
         var allTokens = IntStream.range(0, RustTokenConstants.NUMBER_DIFF_TOKENS).boxed().toList();
         allTokens = new ArrayList<>(allTokens);
 
@@ -139,8 +136,9 @@ class RustFrontendTest {
 
         if (!allTokens.isEmpty()) {
             var notFoundTypes = allTokens.stream().map(type -> new RustToken(type, EMPTY_STRING, NOT_SET, NOT_SET, NOT_SET).type2string()).toList();
-            fail("Some %d token types were not found in the complete code example '%s':\n%s".formatted(notFoundTypes.size(), fileName,
+            logger.error("Some %d token types were not found in the complete code example '%s':\n%s".formatted(notFoundTypes.size(), fileName,
                     notFoundTypes));
+            assertTrue(false);
         }
     }
 

languages/scala/src/main/scala/de/jplag/scala/Language.scala

@@ -1,8 +1,9 @@
 package de.jplag.scala
 
-import de.jplag.TokenList
+import de.jplag.Token
 
 import java.io.File
+import scala.collection.JavaConverters._
 
 import org.kohsuke.MetaInfServices
 
@@ -18,7 +19,7 @@ class Language extends de.jplag.Language {
 
   override def minimumTokenMatch = 8
 
-  override def parse(dir: File, files: Array[String]): TokenList = this.parser.parse(dir, files)
+  override def parse(dir: File, files: Array[String]): java.util.List[Token] = this.parser.parse(dir, files).asJava
 
   override def hasErrors: Boolean = this.parser.hasErrors