Merge branch 'master' into table

# Conflicts:
#	jplag/src/main/java/jplag/reporting/Report.java

.github/workflows/maven.yml

@@ -18,7 +18,7 @@ jobs:
     - name: Set up JDK
       uses: actions/setup-java@v2
       with:
-        java-version: 11
+        java-version: 14
         distribution: 'zulu'
         
     - name: Run Maven Install

README.md

@@ -22,19 +22,20 @@ You'll find the generated JAR with all dependencies in  `jplag/target`.
 
 > Note: The following list is incomplete and gives a rough overview of the changes. In case you depend on missing features we refer to the [legacy release v2.12.1](https://github.com/jplag/jplag/releases/tag/v2.12.1-SNAPSHOT) and the [legacy branch](https://github.com/jplag/jplag/tree/legacy).
 
-* Removed folders related to (deprecated) web services: `adminTool`, `atujplag`, `homepage`, `maven-libs`, `webService`, and `wsClient`
-* Deleted unnecessary resources from `jplag/src/main/resources`
-* All Cluster-related code fragments are commented and marked as `TODO`
-* The new `JPlagOptions` class replaces all previous Options-related classes and manages all available program options
-* Added the `argparse4j` package to properly parse CLI arguments
+* Java API to allow 3rd party integration (see Java API usage below).
+* Result clustering is currently not supported (but might be restored).
+* All comparisons are now sorted by average similarity. Maximum similarity is currently not supported (but might be restored).
+* The CLI interface is now properly defined based on `argparse4j`. The CLI interface was slightly changed, a detailed description of all available options of the new CLI can be found below.
+* The following comparison modes are no longer supported: **experimental**, **external**, **special** and **revision**.
 * Renamed the `Program` class to `JPlag`. It contains the main logic, the parsed submissions, and delegates the comparison of files to one of the new `ComparisonStrategy` implementing classes.
-* Deleted the **experimental** comparison mode. The **external** and **special** comparison strategies are commented out and marked as `TODO`. The **Normal** and **Revision** strategies should work as intended.
-* The new `JPlagResult` class is supposed to bundle all results of a plagiarism detection run. An instance of this class can optionally be passed to the new `Report` class to generate web pages of the results
-* While re-implementing the CLI, we renamed/removed some arguments to adapt the CLI to the new code structure. A detailed description of all available options of the new CLI can be found below.
+* The new `JPlagOptions` class replaces all previous Options-related classes and manages all available program options.
+* The new `JPlagResult` class bundles all results of a plagiarism detection run. An instance of this class can optionally be passed to the new `Report` class to generate web pages for the results
 * The new `JPlagComparison` class replaces the old `AllMatches` class.
-* We removed `AvgComparator`, `AvgReversedComparator`, `MaxComparator`, `MaxReversedComparator`, `MinComparator`, and `MinReversedComparator` from the `JPlagComparison` (previously `AllMatches`) class. All comparisons are now sorted by average similarity. The `JPlagResult` and `JPlagComparison` classes should make adding a custom sorting logic very straightforward if that's required.
+* We removed `AvgComparator`, `AvgReversedComparator`, `MaxComparator`, `MaxReversedComparator`, `MinComparator`, and `MinReversedComparator` from the `JPlagComparison` (previously `AllMatches`) class. The `JPlagResult` and `JPlagComparison` classes should make adding a custom sorting logic very straightforward if that's required.
+* Removed deprecated web service: `adminTool`, `atujplag`, `homepage`, `maven-libs`, `webService`, and `wsClient`
 
 ## Usage
+JPlag can either be used via the CLI or directly via its Java API.
 
 ### CLI
 

jplag.frontend.java-1.9/src/main/java/jplag/java19/JavacAdapter.java

@@ -48,6 +48,7 @@ public class JavacAdapter {
         int errors = 0;
         for (Diagnostic<?> diagItem : diagListen.getDiagnostics()) {
             if (diagItem.getKind() == javax.tools.Diagnostic.Kind.ERROR) {
+                System.out.println(diagItem); // TODO TS: Implement proper logging.
                 errors++;
             }
         }

jplag.parent/pom.xml

@@ -65,8 +65,8 @@
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <project.build.source>11</project.build.source>
-    <project.build.target>11</project.build.target>
+    <project.build.source>14</project.build.source>
+    <project.build.target>14</project.build.target>
     <project.build.optimize>true</project.build.optimize>
   </properties>
 

jplag/src/main/java/jplag/JPlag.java

@@ -82,7 +82,6 @@ public class JPlag implements ProgramI {
 
         // 3. Compare valid submissions:
         errorVector = null; // errorVector is not needed anymore
-        System.gc();
         JPlagResult result = comparisonStrategy.compareSubmissions(submissions, baseCodeSubmission);
         return result;
     }
@@ -304,15 +303,11 @@ public class JPlag implements ProgramI {
     private void parseAllSubmissions(Vector<Submission> submissions, Submission baseCodeSubmission) throws ExitException {
         try {
             parseSubmissions(submissions);
-            System.gc();
             parseBaseCodeSubmission(baseCodeSubmission);
         } catch (OutOfMemoryError e) {
-            System.gc();
-
             throw new ExitException("Out of memory during parsing of submission \"" + currentSubmissionName + "\"");
         } catch (Throwable e) {
             e.printStackTrace();
-
             throw new ExitException("Unknown exception during parsing of " + "submission \"" + currentSubmissionName + "\"");
         }
     }
@@ -438,16 +433,16 @@ public class JPlag implements ProgramI {
         excludedFileNames = new HashSet<>();
 
         try {
-            BufferedReader in = new BufferedReader(new FileReader(options.getExclusionFileName()));
+            BufferedReader reader = new BufferedReader(new FileReader(options.getExclusionFileName()));
             String line;
 
-            while ((line = in.readLine()) != null) {
+            while ((line = reader.readLine()) != null) {
                 excludedFileNames.add(line.trim());
             }
 
-            in.close();
-        } catch (IOException e) {
-            System.out.println("Could not read exclusion file: " + options.getExclusionFileName());
+            reader.close();
+        } catch (IOException exception) {
+            System.out.println("Could not read exclusion file: "+ exception.getMessage());
         }
 
         if (options.getVerbosity() == LONG) {

jplag/src/main/java/jplag/JPlagResult.java

@@ -52,8 +52,8 @@ public class JPlagResult {
         this.durationInMillis = durationInMillis;
         this.numberOfSubmissions = numberOfSubmissions;
         this.options = options;
-
-        this.similarityDistribution = calculateSimilarityDistribution(comparisons);
+        similarityDistribution = calculateSimilarityDistribution(comparisons);
+        comparisons.sort((first, second) -> Float.compare(second.percent(), first.percent())); // Sort by percentage (descending).
     }
 
     /**
@@ -68,6 +68,9 @@ public class JPlagResult {
         return similarityDistribution;
     }
 
+    /**
+     * @return a list of comparisons sorted by percentage (descending).
+     */
     public List<JPlagComparison> getComparisons() {
         return comparisons;
     }

jplag/src/main/java/jplag/reporting/Report.java

@@ -29,7 +29,8 @@ import jplag.TokenList;
  */
 public class Report {
 
-    private final String[] pics = {"forward.gif", "back.gif"};
+    private static final String CSV_FILE = "matches_avg.csv";
+    private static final String[] PICS = {"forward.gif", "back.gif"};
 
     private final Map<JPlagComparison, Integer> comparisonToIndex = new HashMap<>();
     private final Messages msg;
@@ -38,6 +39,8 @@ public class Report {
     private JPlagResult result;
 
     private int currentComparisonIndex = 0;
+    private int matchWritingProgess = 0;
+    private int matchesWritten = 0;
 
     public Report(File reportDir) throws ExitException {
         this.reportDir = reportDir;
@@ -48,12 +51,11 @@ public class Report {
 
     public void writeResult(JPlagResult result) throws ExitException {
         this.result = result;
-
+        System.out.println("Writing report...");
         writeIndex();
-
         copyStaticFiles();
-
         writeMatches(result.getComparisons());
+        System.out.println("Report exported to " + reportDir.getAbsolutePath());
     }
 
     /*
@@ -182,6 +184,15 @@ public class Report {
         htmlFile.println("</TABLE>");
     }
 
+    private synchronized void reportMatchWritingProgress(List<JPlagComparison> comparisons) {
+        matchesWritten++;
+        int currentProgress = 20 * matchesWritten / comparisons.size() * 5; // report every 5% step
+        if (currentProgress > matchWritingProgess) {
+            System.out.println("Writing matches: " + currentProgress + "%");
+            matchWritingProgess = currentProgress;
+        }
+    }
+
     /**
      * Make sure the report directory exists, is a directory and has write access.
      */
@@ -275,7 +286,7 @@ public class Report {
                 if (start.file.equals(files[fileIndex]) && text[fileIndex] != null) {
                     String tmp = "<FONT color=\"" + Color.getHexadecimalValue(x) + "\">" + (j == 1 ? "<div style=\"position:absolute;left:0\">" : "")
                             + "<A HREF=\"javascript:ZweiFrames('match" + i + "-" + (1 - j) + ".html#" + x + "'," + (3 - j) + ",'match" + i
-                            + "-top.html#" + x + "',1)\"><IMG SRC=\"" + pics[j] + "\" ALT=\"other\" " + "BORDER=\"0\" ALIGN=\""
+                            + "-top.html#" + x + "',1)\"><IMG SRC=\"" + PICS[j] + "\" ALT=\"other\" " + "BORDER=\"0\" ALIGN=\""
                             + (j == 0 ? "right" : "left") + "\"></A>" + (j == 1 ? "</div>" : "") + "<B>";
                     // position the icon and the beginning of the colorblock
                     markupList.put(new MarkupText(fileIndex, start.getLine() - 1, start.getColumn() - 1, tmp, true), null);
@@ -400,11 +411,9 @@ public class Report {
         writeIndexBegin(htmlFile, msg.getString("Report.Search_Results"));
         writeDistribution(htmlFile);
 
-        String csvFile = "matches_avg.csv";
-
-        writeLinksToComparisons(htmlFile, "<H4>" + msg.getString("Report.MatchesAvg"), csvFile);
+        writeLinksToComparisons(htmlFile, "<H4>" + msg.getString("Report.MatchesAvg"), CSV_FILE);
 
-        writeMatchesCSV(csvFile);
+        writeMatchesCSV(CSV_FILE);
 
         writeIndexEnd(htmlFile);
 
@@ -541,7 +550,7 @@ public class Report {
     }
 
     private void writeLinksToComparisons(HTMLFile htmlFile, String headerStr, String csvFile) {
-        List<JPlagComparison> comparisons = result.getComparisons();
+        List<JPlagComparison> comparisons = result.getComparisons(); // should be already sorted!
 
         htmlFile.println(headerStr + " (<a href=\"help-sim-" + "en" // Country tag
                 + ".html\"><small><font color=\"#000088\">" + msg.getString("Report.WhatIsThis") + "</font></small></a>):</H4>");
@@ -623,13 +632,13 @@ public class Report {
     }
 
     private void writeMatches(List<JPlagComparison> comparisons) {
-        comparisons.forEach(comparison -> {
+        comparisons.parallelStream().forEach(comparison -> {
             try {
-                int i = getComparisonIndex(comparison);
-                writeMatch(comparison, i);
-            } catch (ExitException e) {
-                e.printStackTrace();
+                writeMatch(comparison, getComparisonIndex(comparison));
+            } catch (ExitException exception) {
+                exception.printStackTrace();
             }
+            reportMatchWritingProgress(comparisons);
         });
     }
 
@@ -687,7 +696,7 @@ public class Report {
                 if (start.file.equals(files[y]) && text[y] != null) {
                     hilf = "<FONT color=\"" + Color.getHexadecimalValue(x) + "\">" + (j == 1 ? "<div style=\"position:absolute;left:0\">" : "")
                             + "<A HREF=\"javascript:ZweiFrames('match" + i + "-" + (1 - j) + ".html#" + x + "'," + (3 - j) + ",'match" + i
-                            + "-top.html#" + x + "',1)\"><IMG SRC=\"" + pics[j] + "\" ALT=\"other\" " + "BORDER=\"0\" ALIGN=\""
+                            + "-top.html#" + x + "',1)\"><IMG SRC=\"" + PICS[j] + "\" ALT=\"other\" " + "BORDER=\"0\" ALIGN=\""
                             + (j == 0 ? "right" : "left") + "\"></A>" + (j == 1 ? "</div>" : "") + "<B>";
                     // position the icon and the beginning of the colorblock
                     if (text[y][start.getLine() - 1].endsWith("</FONT>")) {

jplag/src/test/java/jplag/NewJavaFeaturesTest.java

@@ -0,0 +1,28 @@
+package jplag;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class NewJavaFeaturesTest extends TestBase {
+    private static final String EXCLUSION_FILE_NAME = "blacklist.txt";
+    private static final String ROOT_DIRECTORY = "NewJavaFeatures";
+
+    @Test
+    public void testJavaFeatureDuplicates() throws ExitException {
+        JPlagResult result = runJPlagWithExclusionFile(ROOT_DIRECTORY, EXCLUSION_FILE_NAME);
+        
+        // Two submissions, one comparison with 100% match:
+        assertEquals(2, result.getNumberOfSubmissions());
+        assertEquals(1, result.getComparisons().size());
+        var comparison = result.getComparisons().get(0);
+        assertEquals(100, comparison.percent(), Double.MIN_NORMAL);
+        
+        // Five matches that each start at the same position:
+        assertEquals(4, comparison.matches.size());
+        for (Match match : comparison.matches) {
+            assertEquals(match.startA, match.startB);
+        }
+    }
+
+}

jplag/src/test/java/jplag/TestBase.java

@@ -1,15 +1,27 @@
 package jplag;
 
+import java.util.function.Consumer;
+
 import jplag.options.JPlagOptions;
 import jplag.options.LanguageOption;
 
 public abstract class TestBase {
 
+    private static final String BASE_PATH = "src/test/resources/samples";
+
+    protected JPlagResult runJPlagWithExclusionFile(String testSampleName, String exclusionFileName) throws ExitException {
+        String blackList = String.format(BASE_PATH + "/%s/%s", testSampleName, exclusionFileName);
+        return runJPlagWithOptions(testSampleName, options -> options.setExclusionFileName(blackList));
+    }
+
     protected JPlagResult runJPlagWithDefaultOptions(String testSampleName) throws ExitException {
-        JPlagOptions options = new JPlagOptions(
-                String.format("src/test/resources/samples/%s", testSampleName),
-                LanguageOption.JAVA_1_9);
+        return runJPlagWithOptions(testSampleName, options -> {});
+    }
 
+    protected JPlagResult runJPlagWithOptions(String testSampleName, Consumer<JPlagOptions> customization) throws ExitException {
+        JPlagOptions options = new JPlagOptions(String.format(BASE_PATH + "/%s", testSampleName), LanguageOption.JAVA_1_9);
+        options.setDebugParser(true);
+        customization.accept(options);
         JPlag jplag = new JPlag(options);
         return jplag.run();
     }

jplag/src/test/resources/samples/NewJavaFeatures/A/Java10.java

@@ -0,0 +1,19 @@
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+public class Java10 {
+    
+    List<Optional<String>> test;
+    
+    public Java10() {
+        // Var type for local variables (JEP 286):
+        test = new ArrayList<>();
+        var variable = test;
+        var length = variable.toString().length();
+        test = variable;
+        test.add(Optional.of(variable.toString()));
+        System.out.println(length);
+    }
+
+}

jplag/src/test/resources/samples/NewJavaFeatures/A/Java11.java

@@ -0,0 +1,38 @@
+import java.net.Authenticator;
+import java.net.InetSocketAddress;
+import java.net.ProxySelector;
+import java.net.http.HttpClient;
+import java.net.http.HttpClient.Redirect;
+import java.net.http.HttpClient.Version;
+import java.time.Duration;
+
+public class Java11 {
+
+    public static void main(String[] args) {
+        // Var type allowed in lambda paramters (JEP 323):
+        Cal cal = (var a, var b) -> a + b;
+        int result = cal.sum(10, 20);
+        System.out.println(result);
+        
+        // New HTTP Client API:
+        HttpClient client = HttpClient.newBuilder()
+                .version(Version.HTTP_1_1)
+                .followRedirects(Redirect.NORMAL)
+                .connectTimeout(Duration.ofSeconds(20))
+                .proxy(ProxySelector.of(new InetSocketAddress("proxy.example.com", 80)))
+                .authenticator(Authenticator.getDefault())
+                .build();
+        System.out.println(client);
+        
+        // New string methods:
+        String text = "ABCD";
+        text.repeat(2);
+        text.isBlank();
+        text.strip();
+        text.lines();
+    }
+    
+    interface Cal {
+        int sum(int a, int b);
+    }
+}

jplag/src/test/resources/samples/NewJavaFeatures/A/Java12.java

@@ -0,0 +1,17 @@
+import java.text.NumberFormat;
+import java.util.Locale;
+
+public class Java12 {
+    public static void main(String[] args) {
+        // New String methods:
+        String text = "ABCD";
+        text.indent(5);
+        text.transform(it -> it + "!");
+        
+        // Compact number format:
+        NumberFormat fmt = NumberFormat.getCompactNumberInstance(
+                new Locale("hi", "IN"), NumberFormat.Style.SHORT);
+        String result = fmt.format(1000);
+        System.out.println(result);
+    }
+}

jplag/src/test/resources/samples/NewJavaFeatures/A/Java14.java

@@ -0,0 +1,17 @@
+
+public enum Java14 {
+    MONDAY,
+    TUESDAY,
+    SATURDAY,
+    SUNDAY;
+
+    public static void main(String[] args) {
+        // Advanced switch expressions (JEP 361):
+        boolean isWeekend = switch (MONDAY) {
+        case SATURDAY, SUNDAY -> true;
+        default -> false;
+        };
+        System.out.println(isWeekend);
+    }
+
+}

jplag/src/test/resources/samples/NewJavaFeatures/A/Java16.java

@@ -0,0 +1,25 @@
+
+public class Java16 {
+    public static void main(String[] args) {
+        // Text blocks (JEP 378): 
+        String query = """
+                SELECT "EMP_ID", "LAST_NAME"
+                FROM "EMPLOYEE_TB";
+                """;
+        System.out.println(query);
+        
+        // Pattern matching for instanceof (JEP 394):
+        String x = "123";
+        if (x instanceof String s) { String a = s; }
+        
+        // Record types (JEP 395):
+        Point p = new Point(1, 2);
+        System.out.println(p.add());
+    }
+    
+    record Point(int x, int y) {
+        public int add() {
+            return x + y;
+        }
+    }
+}

jplag/src/test/resources/samples/NewJavaFeatures/B/Java10.java

@@ -0,0 +1,19 @@
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+public class Java10 {
+    
+    List<Optional<String>> test;
+    
+    public Java10() {
+        // Var type for local variables (JEP 286):
+        test = new ArrayList<>();
+        var variable = test;
+        int length = variable.toString().length();
+        test = variable;
+        test.add(Optional.of(variable.toString()));
+        System.out.println(length);
+    }
+
+}

jplag/src/test/resources/samples/NewJavaFeatures/B/Java11.java

@@ -0,0 +1,38 @@
+import java.net.Authenticator;
+import java.net.InetSocketAddress;
+import java.net.ProxySelector;
+import java.net.http.HttpClient;
+import java.net.http.HttpClient.Redirect;
+import java.net.http.HttpClient.Version;
+import java.time.Duration;
+
+public class Java11 {
+
+    public static void main(String[] args) {
+        // Var type allowed in lambda paramters (JEP 323):
+        Cal cal = (a, b) -> a + b;
+        int result = cal.sum(10, 20);
+        System.out.println(result);
+        
+        // New HTTP Client API:
+        HttpClient client = HttpClient.newBuilder()
+                .version(Version.HTTP_1_1)
+                .followRedirects(Redirect.ALWAYS)
+                .connectTimeout(Duration.ofSeconds(15))
+                .proxy(ProxySelector.of(new InetSocketAddress("proxy.example.de", 90)))
+                .authenticator(Authenticator.getDefault())
+                .build();
+        System.out.println(client);
+        
+        // New string methods:
+        String text = "ABCD";
+        text.repeat(4);
+        text.isBlank();
+        text.strip();
+        text.lines();
+    }
+    
+    interface Cal {
+        int sum(int a, int b);
+    }
+}

jplag/src/test/resources/samples/NewJavaFeatures/B/Java12.java

@@ -0,0 +1,17 @@
+import java.text.NumberFormat;
+import java.util.Locale;
+
+public class Java12 {
+    public static void main(String[] args) {
+        // New String methods:
+        String text = "ABCD";
+        text.indent(2);
+        text.transform(it -> it + "?");
+        
+        // Compact number format:
+        NumberFormat fmt = NumberFormat.getCompactNumberInstance(
+                new Locale("hi", "IN"), NumberFormat.Style.SHORT);
+        String result = fmt.format(1000);
+        System.out.println(result);
+    }
+}

jplag/src/test/resources/samples/NewJavaFeatures/B/Java14.java

@@ -0,0 +1,17 @@
+
+public enum Java14 {
+    MONDAY,
+    TUESDAY,
+    SATURDAY,
+    SUNDAY;
+
+    public static void main(String[] args) {
+        // Advanced switch expressions (JEP 361):
+        boolean isWeekend = switch (MONDAY) {
+        case FRIDAY, SATURDAY, SUNDAY -> true;
+        default -> false;
+        };
+        System.out.println(isWeekend);
+    }
+
+}

jplag/src/test/resources/samples/NewJavaFeatures/B/Java16.java

@@ -0,0 +1,25 @@
+
+public class Java16 {
+    public static void main(String[] args) {
+        // Text blocks (JEP 378): 
+        String query = """
+                SELECT "EMPLOYEE_ID", "NAME"
+                FROM "EMPLOYEE_TABLE";
+                """;
+        System.out.println(query);
+        
+        // Pattern matching for instanceof (JEP 394):
+        String x = "123";
+        if (x instanceof String text) { String answer = text; }
+        
+        // Record types (JEP 395):
+        Point q = new Point(1, 2);
+        System.out.println(q.add());
+    }
+    
+    record Point(int x, int y) {
+        public int add() {
+            return y + x;
+        }
+    }
+}

jplag/src/test/resources/samples/NewJavaFeatures/blacklist.txt

@@ -0,0 +1 @@
+Java16.java