Add support for suffixed files back into TokenPrinter, eliminate unused parameters

jplag.frontend-utils/src/main/java/de/jplag/TokenPrinter.java

@@ -1,12 +1,13 @@
 package de.jplag;
 
-import static java.util.stream.Collectors.toMap;
-
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
-import java.util.*;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
@@ -47,52 +48,27 @@ public final class TokenPrinter {
      * Creates a string representation of a set of files line by line and adds the tokens under the lines.
      * @param tokens is the set of tokens parsed from the files.
      * @param rootDirectory is the common rootDirectory of the files.
-     * @param fileNames is a collection of the file names.
-     * @param suffix is the optional view file suffix.
-     * @return the string representation.
-     */
-    public static String printTokens(TokenList tokens, File rootDirectory, Collection<String> fileNames, Optional<String> suffix) {
-        Collection<File> files = fileNames.stream().map(name -> new File(rootDirectory, name)).toList();
-        return printTokens(tokens, files, rootDirectory, suffix);
-    }
-
-    /**
-     * Creates a string representation of a set of files line by line and adds the tokens under the lines.
-     * @param tokens is the set of tokens parsed from the files.
-     * @param rootDirectory is the common rootDirectory of the files.
-     * @param fileNames is a collection of the file names.
-     * @return the string representation.
-     */
-    public static String printTokens(TokenList tokens, File rootDirectory, Collection<String> fileNames) {
-        return printTokens(tokens, rootDirectory, fileNames, Optional.empty());
-    }
-
-    /**
-     * Creates a string representation of a collection of files line by line and adds the tokens under the lines.
-     * @param tokens is the set of tokens parsed from the files.
-     * @param files are the parsed files.
      * @return the string representation.
      */
-    public static String printTokens(TokenList tokens, Collection<File> files, File rootDirectory) {
-        return printTokens(tokens, files, rootDirectory, Optional.empty());
+    public static String printTokens(TokenList tokens, File rootDirectory) {
+        return printTokens(tokens, rootDirectory, Optional.empty());
     }
 
     /**
      * Creates a string representation of a collection of files line by line and adds the tokens under the lines.
      * @param tokenList is the set of tokens parsed from the files.
-     * @param files are the parsed files.
      * @param rootDirectory is the common directory of the files.
      * @param suffix is the optional view file suffix.
      * @return the string representation.
      */
-    public static String printTokens(TokenList tokenList, Collection<File> files, File rootDirectory, Optional<String> suffix) {
+    public static String printTokens(TokenList tokenList, File rootDirectory, Optional<String> suffix) {
         PrinterOutputBuilder builder = new PrinterOutputBuilder();
         Map<String, List<Token>> fileToTokens = groupTokensByFile(tokenList);
 
         fileToTokens.forEach((String fileName, List<Token> fileTokens) -> {
             builder.append(fileName);
 
-            var lineDatas = getLineData(fileTokens, rootDirectory);
+            var lineDatas = getLineData(fileTokens, rootDirectory, suffix);
             lineDatas.forEach(lineData -> {
                 builder.setLine(lineData.lineNumber());
 
@@ -135,12 +111,15 @@ public final class TokenPrinter {
         return builder.toString();
     }
 
-    private static List<LineData> getLineData(List<Token> fileTokens, File root) {
+    private static List<LineData> getLineData(List<Token> fileTokens, File root, Optional<String> suffix) {
         // We expect that all fileTokens share the same Token.file!
 
         String fileName = fileTokens.get(0).getFile();
         // handle 'files as submissions' mode
         File file = fileName.isEmpty() ? root : new File(root, fileName);
+        if (suffix.isPresent()) {
+            file = new File(file.getPath() + suffix.get());
+        }
 
         // Sort tokens by file and line -> tokens can be processed without any further checks
         List<String> lines = linesFromFile(file);
@@ -164,13 +143,6 @@ public final class TokenPrinter {
         return token.getLength() <= 1 && INDICATE_TINY_TOKEN ? token.toString().toLowerCase() : token.toString();
     }
 
-    /**
-     * Turns a list of files into a map which maps the file names to a list of the contained lines of that file.
-     */
-    private static Map<String, List<String>> readFiles(Collection<File> allFiles) {
-        return allFiles.stream().collect(toMap(File::getName, TokenPrinter::linesFromFile));
-    }
-
     /**
      * Parses a file and returns a list of the contained lines.
      */

jplag.frontend-utils/src/test/java/de/jplag/TokenPrinterTest.java

@@ -5,15 +5,18 @@ import static de.jplag.simple.TestTokenConstants.STRING;
 import static org.junit.jupiter.api.Assertions.fail;
 
 import java.nio.file.Path;
-import java.util.List;
 
 import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-public class TokenPrinterTest {
+class TokenPrinterTest {
 
     private static final Path TEST_FILE_LOCATION = Path.of("src", "test", "resources", "de", "jplag", "samples");
     private static final String TEST_FILE_NAME = "TokenPrinterTest.txt";
-    public static final String TOKEN_SEPARATOR = "(\\|)?\s*\\|";
+    private static final String TOKEN_SEPARATOR = "(\\|)?\s*\\|";
+
+    private static final Logger logger = LoggerFactory.getLogger(TokenPrinterTest.class);
 
     /**
      * Tests the function of the TokenPrinter
@@ -55,8 +58,8 @@ public class TokenPrinterTest {
 
         tokens.addToken(new TestToken(STRING, TEST_FILE_NAME, 100, 1, 1));
 
-        String output = TokenPrinter.printTokens(tokens, TEST_FILE_LOCATION.toFile(), List.of(TEST_FILE_NAME));
-        System.out.print(output); // no additional newline required
+        String output = TokenPrinter.printTokens(tokens, TEST_FILE_LOCATION.toFile());
+        logger.debug(output); // no additional newline required
 
         testOutputCorrectness(TEST_FILE_NAME, tokens, output);
     }

jplag.frontend.csharp-6/src/test/java/de/jplag/csharp/MinimalCSharpFrontendTest.java

@@ -7,7 +7,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.File;
 import java.nio.file.Path;
-import java.util.Arrays;
 import java.util.List;
 
 import org.junit.jupiter.api.BeforeEach;
@@ -46,7 +45,7 @@ class MinimalCSharpFrontendTest {
         // Parse test input
         String[] input = new String[] {TEST_SUBJECT};
         TokenList result = frontend.parse(baseDirectory, input);
-        logger.info(TokenPrinter.printTokens(result, baseDirectory, Arrays.asList(input)));
+        logger.info(TokenPrinter.printTokens(result, baseDirectory));
 
         // Compare parsed tokens:
         assertEquals(expectedToken.size(), result.size());

jplag.frontend.emf-metamodel-dynamic/src/test/java/de/jplag/emf/dynamic/MinimalDynamicMetamodelTest.java

@@ -6,8 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.File;
 import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.List;
 import java.util.Optional;
 
 import org.junit.jupiter.api.AfterEach;
@@ -40,8 +38,7 @@ class MinimalDynamicMetamodelTest {
     @Test
     void testBookstoreMetamodels() {
         TokenList result = frontend.parse(baseDirectory, TEST_SUBJECTS);
-        List<String> treeViewFiles = Arrays.stream(TEST_SUBJECTS).map(it -> it + Language.VIEW_FILE_SUFFIX).toList();
-        logger.debug(TokenPrinter.printTokens(result, baseDirectory, treeViewFiles, Optional.of(Language.VIEW_FILE_SUFFIX)));
+        logger.debug(TokenPrinter.printTokens(result, baseDirectory, Optional.of(Language.VIEW_FILE_SUFFIX)));
         logger.info(("Dynamic token set: " + DynamicMetamodelTokenConstants.getTokenStrings()));
         logger.info("parsed tokens: " + result.allTokens().toString());
         assertEquals(7, DynamicMetamodelTokenConstants.getTokenStrings().size());

jplag.frontend.emf-metamodel/src/test/java/de/jplag/emf/MinimalMetamodelTest.java

@@ -1,14 +1,11 @@
 package de.jplag.emf;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertIterableEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.*;
 
 import java.io.File;
 import java.lang.reflect.Field;
 import java.nio.file.Path;
 import java.util.Arrays;
-import java.util.List;
 import java.util.Optional;
 
 import org.junit.jupiter.api.AfterEach;
@@ -41,9 +38,8 @@ class MinimalMetamodelTest {
     @Test
     void testBookstoreMetamodels() {
         TokenList result = frontend.parse(baseDirectory, TEST_SUBJECTS);
-        List<String> treeViewFiles = Arrays.stream(TEST_SUBJECTS).map(it -> it + Language.VIEW_FILE_SUFFIX).toList();
 
-        logger.debug(TokenPrinter.printTokens(result, baseDirectory, treeViewFiles, Optional.of(Language.VIEW_FILE_SUFFIX)));
+        logger.debug(TokenPrinter.printTokens(result, baseDirectory, Optional.of(Language.VIEW_FILE_SUFFIX)));
         Field[] fields = MetamodelTokenConstants.class.getFields();
         var constants = Arrays.stream(fields).map(Field::getName).filter(it -> !it.equals("NUM_DIFF_TOKENS")).toList();
         logger.info(("Handcrafted token set: " + constants));

jplag.frontend.golang/src/test/java/de/jplag/golang/GoFrontendTest.java

@@ -59,7 +59,7 @@ class GoFrontendTest {
     void parseTestFiles() {
         for (String fileName : testFiles) {
             TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
-            String output = TokenPrinter.printTokens(tokens, testFileLocation, List.of(fileName));
+            String output = TokenPrinter.printTokens(tokens, testFileLocation);
             logger.info(output);
 
             testSourceCoverage(fileName, tokens);

jplag.frontend.kotlin/src/test/java/de/jplag/kotlin/KotlinFrontendTest.java

@@ -59,7 +59,7 @@ class KotlinFrontendTest {
     void parseTestFiles() {
         for (String fileName : testFiles) {
             TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
-            String output = TokenPrinter.printTokens(tokens, testFileLocation, List.of(fileName));
+            String output = TokenPrinter.printTokens(tokens, testFileLocation);
             logger.info(output);
 
             testSourceCoverage(fileName, tokens);

jplag.frontend.rlang/src/test/java/de/jplag/rlang/RFrontendTest.java

@@ -47,7 +47,7 @@ class RFrontendTest {
     void parseTestFiles() {
         for (String fileName : testFiles) {
             TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
-            String output = TokenPrinter.printTokens(tokens, testFileLocation, List.of(fileName));
+            String output = TokenPrinter.printTokens(tokens, testFileLocation);
             logger.info(output);
 
             testSourceCoverage(fileName, tokens);

jplag.frontend.scala/src/test/java/de/jplag/scala/ScalaFrontendTest.java

@@ -60,7 +60,7 @@ class ScalaFrontendTest {
     void parseTestFiles() {
         for (String fileName : testFiles) {
             TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
-            String output = TokenPrinter.printTokens(tokens, testFileLocation, List.of(fileName));
+            String output = TokenPrinter.printTokens(tokens, testFileLocation);
             logger.info(output);
 
             if (fileName.equals(COMPLETE_TEST_FILE)) {

jplag.frontend.text/src/test/java/jplag/text/TextFrontendTest.java

@@ -5,10 +5,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.File;
 import java.nio.file.Path;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.stream.StreamSupport;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -41,11 +39,11 @@ class TextFrontendTest {
         // Parse test input
         String[] input = new String[] {TEST_SUBJECT};
         TokenList result = frontend.parse(baseDirectory, input);
-        logger.info(TokenPrinter.printTokens(result, baseDirectory, Arrays.asList(input)));
+        logger.info(TokenPrinter.printTokens(result, baseDirectory));
 
         // Compare parsed tokens:
         Map<Integer, Token> tokenTypes = new HashMap<>();
-        StreamSupport.stream(result.allTokens().spliterator(), false).forEach(it -> tokenTypes.put(it.getType(), it));
+        result.allTokens().forEach(it -> tokenTypes.put(it.getType(), it));
 
         assertEquals(293, result.size());
         assertEquals(156, tokenTypes.values().size());

jplag/src/main/java/de/jplag/Submission.java

@@ -204,7 +204,7 @@ public class Submission implements Comparable<Submission> {
      * @return the annotated code as string.
      */
     public String getTokenAnnotatedSourceCode() {
-        return TokenPrinter.printTokens(tokenList, files, submissionRootFile);
+        return TokenPrinter.printTokens(tokenList, submissionRootFile);
     }
 
     @Override

jplag/src/test/java/de/jplag/special/TokenPrinterTest.java

@@ -79,7 +79,7 @@ public class TokenPrinterTest extends TestBase {
         System.out.println(LINE);
         System.out.println(LARGE_SPACE + submission.getName());
         System.out.println(LINE);
-        System.out.println(TokenPrinter.printTokens(submission.getTokenList(), submission.getFiles(), submission.getRoot()));
+        System.out.println(TokenPrinter.printTokens(submission.getTokenList(), submission.getRoot()));
     }
 
 }