Merge pull request #401 from jplag/token-printer

Merges the token printer utility for debugging, testing, and frontend creation. Allows printing the parsed lines of codes and the corresponding tokens underneath. Each line of code is printed starting with the line number. Under these lines, the tokens are annotated in the format `|TOKEN|`. The first vertical line marks the token start, while the last vertical line marks the token end. Tokens that are shorter than the name do not end with a vertical line, e.g. `|TOKEN`. Tokens with length 1 or 0 are printed in lower case, e.g. `|token`.

Also adds a special test case to the core project for easy token printing. This is not really a test case per se, but it prints all source code files from the printing folder in the test resources. By default, this test is disabled.

- the token printer is part of the frontend-utils
- the token printer "test" is part of the core test

jplag.frontend-utils/src/main/java/de/jplag/TokenPrinter.java

@@ -0,0 +1,153 @@
+package de.jplag;
+
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toMap;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Utility class for printing JPlag tokens from a submission. Each line of code is printed starting with the line
+ * number. Under these lines the tokens are annotated in the format <code>|TOKEN|</code>. The first vertical line marks
+ * the token start, while the last vertical line marks the token end. Tokens that are shorter than the name do not end
+ * with a vertical line, e.g. <code>|TOKEN</code>. Tokens with length 1 or 0 are printed in lower case, e.g.
+ * <code>|token</code>.
+ * @author Timur Saglam
+ */
+public final class TokenPrinter {
+
+    // Representation:
+    private static final String BAR = "|";
+    private static final String TAB = "\t";
+    private static final String SPACE = " ";
+    private static final String TAB_REPLACEMENT = "        "; // might depend on files
+
+    // Configuration:
+    private static final boolean INDICATE_TINY_TOKEN = true; // print token with length <= 1 in lowercase
+    private static final boolean REPLACE_TABS = false;
+
+    private TokenPrinter() {
+        // Utility class, no public constructor.
+    }
+
+    /**
+     * Creates a string representation of a set of files line by line and adds the tokens under the lines.
+     * @param tokens is the set of tokens parsed from the files.
+     * @param directory is the common directory of the files.
+     * @param fileNames is a collection of the file names.
+     * @return the string representation.
+     */
+    public static String printTokens(TokenList tokens, File directory, Collection<String> fileNames) {
+        Collection<File> files = fileNames.stream().map(name -> new File(directory, name)).collect(toList());
+        return printTokens(tokens, files, directory);
+    }
+
+    /**
+     * Creates a string representation of a collection of files line by line and adds the tokens under the lines.
+     * @param tokens is the set of tokens parsed from the files.
+     * @param files are the parsed files.
+     * @return the string representation.
+     */
+    public static String printTokens(TokenList tokens, Collection<File> files, File root) {
+        Map<String, List<String>> filesToLines = readFiles(files);
+        StringBuilder builder = new StringBuilder();
+        int lineIndex = 0;
+        int columnIndex = 1;
+        String file = null; // no file yet
+
+        for (Token token : tokens.allTokens()) {
+            // New code file:
+            if (!token.getFile().equals(file)) {
+                if (file != null) {
+                    builder.append(System.lineSeparator());
+                }
+                file = token.getFile();
+                builder.append(file);
+                lineIndex = 0;
+            }
+
+            // New code line:
+            if (token.getLine() > lineIndex) {
+                lineIndex = token.getLine();
+                columnIndex = 1;
+
+                String fileName = token.getFile().isEmpty() ? root.getName() : token.getFile();
+                String currentLine = filesToLines.get(fileName).get(lineIndex - 1);
+                if (REPLACE_TABS) {
+                    currentLine = currentLine.replace(TAB, TAB_REPLACEMENT);
+                }
+                appendCodeLinePrefix(builder, lineIndex).append(currentLine);
+                appendTokenLinePrefix(builder);
+            }
+
+            // New line if already past token index:
+            if (columnIndex > token.getColumn()) {
+                appendTokenLinePrefix(builder);
+                columnIndex = 1;
+            }
+            // Move to token index:
+            while (columnIndex < token.getColumn()) {
+                builder.append(SPACE);
+                columnIndex++;
+            }
+
+            // Print the actual token:
+            String stringRepresentation = getStringRepresentation(token);
+            builder.append(BAR).append(stringRepresentation);
+            columnIndex += stringRepresentation.length() + 1;
+
+            // Move up to token end:
+            int tokenEndIndex = token.getColumn() + token.getLength() - 1;
+            while (columnIndex < tokenEndIndex) {
+                builder.append(SPACE);
+                columnIndex++;
+            }
+            // Print token end if not already past it:
+            if (columnIndex == tokenEndIndex) {
+                builder.append(BAR);
+                columnIndex++;
+            }
+        }
+        return builder.toString();
+    }
+
+    private static StringBuilder appendCodeLinePrefix(StringBuilder builder, int index) {
+        return builder.append(System.lineSeparator()).append(index + TAB);
+    }
+
+    private static StringBuilder appendTokenLinePrefix(StringBuilder builder) {
+        return builder.append(System.lineSeparator()).append(TAB);
+    }
+
+    /**
+     * Determines the string representation of the token.
+     */
+    private static String getStringRepresentation(Token token) {
+        return token.getLength() <= 1 && INDICATE_TINY_TOKEN ? token.toString().toLowerCase() : token.toString();
+    }
+
+    /**
+     * Turns a list of files into a map which maps the file names to a list of the contained lines of that file.
+     */
+    private static Map<String, List<String>> readFiles(Collection<File> allFiles) {
+        return allFiles.stream().collect(toMap(it -> it.getName(), it -> linesFromFile(it)));
+    }
+
+    /**
+     * Parses a file and returns a list of the contained lines.
+     */
+    private static List<String> linesFromFile(File file) {
+        try {
+            return Files.readAllLines(file.toPath());
+        } catch (IOException exception) {
+            System.err.println("Cannot read " + file.getAbsolutePath() + ":");
+            exception.printStackTrace();
+        }
+        return Collections.emptyList();
+    }
+}

jplag.frontend.java/src/main/java/de/jplag/java/JavaTokenConstants.java

@@ -77,12 +77,12 @@ public interface JavaTokenConstants extends TokenConstants {
     int J_MODULE_END = 65;        // check
 
     // new in 13
-    final static int J_YIELD = 66;
+    int J_YIELD = 66;
 
     // new in 17
-    final static int J_DEFAULT = 67;
-    final static int J_RECORD_BEGIN = 68;
-    final static int J_RECORD_END = 69;
+    int J_DEFAULT = 67;
+    int J_RECORD_BEGIN = 68;
+    int J_RECORD_END = 69;
 
     final static int NUM_DIFF_TOKENS = 70;	    // @formatter:on
 }

jplag/src/main/java/de/jplag/Submission.java

@@ -180,6 +180,14 @@ public class Submission implements Comparable<Submission> {
         this.tokenList = tokenList;
     }
 
+    /**
+     * String representation of the code files contained in this submission, annotated with all tokens.
+     * @return the annotated code as string.
+     */
+    public String getTokenAnnotatedSourcCode() {
+        return TokenPrinter.printTokens(tokenList, files, submissionRootFile);
+    }
+
     @Override
     public String toString() {
         return name;

jplag/src/test/java/de/jplag/special/TokenPrinterTest.java

@@ -0,0 +1,67 @@
+package de.jplag.special;
+
+import static org.junit.Assert.fail;
+
+import java.util.function.Consumer;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import de.jplag.JPlagResult;
+import de.jplag.Submission;
+import de.jplag.TestBase;
+import de.jplag.TokenPrinter;
+import de.jplag.exceptions.ExitException;
+import de.jplag.options.JPlagOptions;
+import de.jplag.options.LanguageOption;
+
+/**
+ * Special test case the does not really test anything but prints the tokens and the corresponding line of code.
+ * @author Timur Saglam
+ */
+public class TokenPrinterTest extends TestBase {
+    private static final String LARGE_SPACE = "   ";
+    private static final String LINE = "------------------";
+
+    private static final int MIN_TOKEN_MATCH = 5;
+    private static final String PRINTER_FOLDER = "PRINTER"; // in the folder 'jplag/src/test/resources/samples'
+
+    @Ignore
+    @Test
+    public void printCPPFiles() {
+        printSubmissions(options -> {
+            options.setLanguageOption(LanguageOption.C_CPP);
+            options.setMinimumTokenMatch(MIN_TOKEN_MATCH); // for printing also allow small files
+        });
+    }
+
+    @Ignore
+    @Test
+    public void printJavaFiles() {
+        printSubmissions(options -> {
+            options.setMinimumTokenMatch(MIN_TOKEN_MATCH); // for printing also allow small files
+        });
+    }
+
+    private void printSubmissions(Consumer<JPlagOptions> optionsCustomization) {
+        try {
+            JPlagResult result = runJPlag(PRINTER_FOLDER, optionsCustomization);
+            for (Submission submission : result.getSubmissions().getSubmissions()) {
+                printSubmission(submission);
+            }
+            System.out.println("JPlag printed " + result.getSubmissions().numberOfSubmissions() + " valid submissions!");
+        } catch (ExitException exception) {
+            System.err.println("JPlag threw Error: " + exception.getMessage());
+            fail();
+        }
+    }
+
+    private void printSubmission(Submission submission) {
+        System.out.println();
+        System.out.println(LINE);
+        System.out.println(LARGE_SPACE + submission.getName());
+        System.out.println(LINE);
+        System.out.println(TokenPrinter.printTokens(submission.getTokenList(), submission.getFiles(), submission.getRoot()));
+    }
+
+}

jplag/src/test/java/de/jplag/VolumeTest.java → jplag/src/test/java/de/jplag/special/VolumeTest.java

@@ -1,4 +1,4 @@
-package de.jplag;
+package de.jplag.special;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -14,6 +14,7 @@ import java.util.Map;
 import org.junit.Ignore;
 import org.junit.Test;
 
+import de.jplag.TestBase;
 import de.jplag.exceptions.ExitException;
 import de.jplag.options.JPlagOptions;
 

jplag/src/test/resources/de/jplag/samples/PRINTER/git.keep

@@ -0,0 +1 @@
+Ensures the folder always exists!