|
|
@@ -1,16 +1,16 @@
|
|
|
package de.jplag;
|
|
|
|
|
|
-import static java.util.stream.Collectors.toMap;
|
|
|
-
|
|
|
import java.io.File;
|
|
|
import java.io.IOException;
|
|
|
import java.nio.file.Files;
|
|
|
import java.nio.file.NoSuchFileException;
|
|
|
-import java.util.Collection;
|
|
|
import java.util.Collections;
|
|
|
import java.util.List;
|
|
|
import java.util.Map;
|
|
|
import java.util.Optional;
|
|
|
+import java.util.stream.Collectors;
|
|
|
+import java.util.stream.IntStream;
|
|
|
+import java.util.stream.Stream;
|
|
|
|
|
|
import org.slf4j.Logger;
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
@@ -29,14 +29,16 @@ public final class TokenPrinter {
|
|
|
private static final String BAR = "|";
|
|
|
private static final String TAB = "\t";
|
|
|
private static final String SPACE = " ";
|
|
|
- public static final String NON_WHITESPACE = "\\S";
|
|
|
- public static final int MIN_PADDING = 1;
|
|
|
+ private static final String NON_WHITESPACE = "\\S";
|
|
|
+ private static final int MIN_PADDING = 1;
|
|
|
private static final int TAB_LENGTH = 8;
|
|
|
private static final String TAB_REPLACEMENT = SPACE.repeat(TAB_LENGTH); // might depend on files
|
|
|
|
|
|
// Configuration:
|
|
|
- private static final boolean INDICATE_TINY_TOKEN = true; // print token with length <= 1 in lowercase
|
|
|
+ private static final boolean INDICATE_TINY_TOKEN = true; // print token with length <= 1 in lowercase
|
|
|
private static final boolean REPLACE_TABS = false;
|
|
|
+ private static final boolean PRINT_EMPTY_LINES = true; // print code lines with no tokens
|
|
|
+ private static final boolean SPACIOUS = true; // print empty line after last token of a line
|
|
|
|
|
|
private TokenPrinter() {
|
|
|
// Utility class, no public constructor.
|
|
|
@@ -45,157 +47,93 @@ public final class TokenPrinter {
|
|
|
/**
|
|
|
* Creates a string representation of a set of files line by line and adds the tokens under the lines.
|
|
|
* @param tokens is the set of tokens parsed from the files.
|
|
|
- * @param directory is the common directory of the files.
|
|
|
- * @param fileNames is a collection of the file names.
|
|
|
- * @param suffix is the optional view file suffix.
|
|
|
+ * @param rootDirectory is the common rootDirectory of the files.
|
|
|
* @return the string representation.
|
|
|
*/
|
|
|
- public static String printTokens(TokenList tokens, File directory, Collection<String> fileNames, Optional<String> suffix) {
|
|
|
- Collection<File> files = fileNames.stream().map(name -> new File(directory, name)).toList();
|
|
|
- return printTokens(tokens, files, directory, suffix);
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Creates a string representation of a set of files line by line and adds the tokens under the lines.
|
|
|
- * @param tokens is the set of tokens parsed from the files.
|
|
|
- * @param directory is the common directory of the files.
|
|
|
- * @param fileNames is a collection of the file names.
|
|
|
- * @return the string representation.
|
|
|
- */
|
|
|
- public static String printTokens(TokenList tokens, File directory, Collection<String> fileNames) {
|
|
|
- Collection<File> files = fileNames.stream().map(name -> new File(directory, name)).toList();
|
|
|
- return printTokens(tokens, files, directory, Optional.empty());
|
|
|
+ public static String printTokens(TokenList tokens, File rootDirectory) {
|
|
|
+ return printTokens(tokens, rootDirectory, Optional.empty());
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* Creates a string representation of a collection of files line by line and adds the tokens under the lines.
|
|
|
- * @param tokens is the set of tokens parsed from the files.
|
|
|
- * @param files are the parsed files.
|
|
|
- * @return the string representation.
|
|
|
- */
|
|
|
- public static String printTokens(TokenList tokens, Collection<File> files, File root) {
|
|
|
- return printTokens(tokens, files, root, Optional.empty());
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Creates a string representation of a collection of files line by line and adds the tokens under the lines.
|
|
|
- * @param tokens is the set of tokens parsed from the files.
|
|
|
- * @param files are the parsed files.
|
|
|
+ * @param tokenList is the set of tokens parsed from the files.
|
|
|
+ * @param rootDirectory is the common directory of the files.
|
|
|
* @param suffix is the optional view file suffix.
|
|
|
* @return the string representation.
|
|
|
*/
|
|
|
- public static String printTokens(TokenList tokens, Collection<File> files, File root, Optional<String> suffix) {
|
|
|
- Map<String, List<String>> filesToLines = readFiles(files);
|
|
|
- StringBuilder builder = new StringBuilder();
|
|
|
- int lineIndex = 0;
|
|
|
- int columnIndex = 1;
|
|
|
- String file = null; // no file yet
|
|
|
- String currentLine = null;
|
|
|
-
|
|
|
- for (Token token : tokens.allTokens()) {
|
|
|
- // New code file:
|
|
|
- if (!token.getFile().equals(file)) {
|
|
|
- if (file != null) {
|
|
|
- builder.append(System.lineSeparator());
|
|
|
- }
|
|
|
- file = token.getFile();
|
|
|
- builder.append(file);
|
|
|
- lineIndex = 0;
|
|
|
- }
|
|
|
+ public static String printTokens(TokenList tokenList, File rootDirectory, Optional<String> suffix) {
|
|
|
+ PrinterOutputBuilder builder = new PrinterOutputBuilder();
|
|
|
+ Map<String, List<Token>> fileToTokens = groupTokensByFile(tokenList);
|
|
|
+
|
|
|
+ fileToTokens.forEach((String fileName, List<Token> fileTokens) -> {
|
|
|
+ builder.append(fileName);
|
|
|
+
|
|
|
+ List<LineData> lineDatas = getLineData(fileTokens, rootDirectory, suffix);
|
|
|
+ lineDatas.forEach(lineData -> {
|
|
|
+ builder.setLine(lineData.lineNumber());
|
|
|
|
|
|
- // New code line:
|
|
|
- if (token.getLine() > lineIndex) {
|
|
|
- lineIndex = token.getLine();
|
|
|
- columnIndex = 1;
|
|
|
+ // Print (prefix and) code line
|
|
|
+ String currentLine = lineData.text();
|
|
|
+ builder.appendCodeLine(currentLine);
|
|
|
|
|
|
- String fileName = token.getFile().isEmpty() ? root.getName() : token.getFile();
|
|
|
- fileName = fileName + suffix.orElse("");
|
|
|
- if (!filesToLines.containsKey(fileName)) {
|
|
|
- throw new IllegalStateException("Cannot find file " + fileName + " in " + filesToLines.keySet());
|
|
|
+ List<Token> tokens = lineData.tokens();
|
|
|
+ if (tokens.isEmpty()) {
|
|
|
+ return;
|
|
|
}
|
|
|
|
|
|
- currentLine = filesToLines.get(fileName).get(lineIndex - 1);
|
|
|
+ builder.appendTokenLinePrefix();
|
|
|
|
|
|
- appendCodeLinePrefix(builder, lineIndex);
|
|
|
- appendCodeLine(builder, currentLine);
|
|
|
- appendTokenLinePrefix(builder, lineIndex);
|
|
|
- }
|
|
|
+ // Print tokens
|
|
|
+ for (Token token : tokens) {
|
|
|
+ // Move to token index, possibly adding a new line:
|
|
|
+ builder.advanceToTokenPosition(currentLine, token.getColumn(), true);
|
|
|
+
|
|
|
+ // Print the actual token:
|
|
|
+ String stringRepresentation = getStringRepresentation(token);
|
|
|
+ builder.append(BAR).append(stringRepresentation);
|
|
|
+
|
|
|
+ // Move up to token end:
|
|
|
+ int tokenEndIndex = token.getColumn() + token.getLength() - 1;
|
|
|
+ builder.advanceToTokenPosition(currentLine, tokenEndIndex, false);
|
|
|
+
|
|
|
+ // Print token end if not already past it:
|
|
|
+ if (builder.positionBeforeOrEqualTo(tokenEndIndex)) {
|
|
|
+ builder.append(BAR);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ builder.appendTokenLineSuffix();
|
|
|
+ builder.advanceToNextLine();
|
|
|
+ });
|
|
|
+ builder.advanceToNextLine();
|
|
|
+ });
|
|
|
|
|
|
- // New line if already past token index:
|
|
|
- if (columnIndex > token.getColumn()) {
|
|
|
- appendTokenLinePrefix(builder, lineIndex);
|
|
|
- columnIndex = 1;
|
|
|
- }
|
|
|
- // Move to token index:
|
|
|
- columnIndex = appendPadding(builder, currentLine, columnIndex, token.getColumn());
|
|
|
-
|
|
|
- // Print the actual token:
|
|
|
- String stringRepresentation = getStringRepresentation(token);
|
|
|
- builder.append(BAR).append(stringRepresentation);
|
|
|
- columnIndex += stringRepresentation.length() + 1;
|
|
|
-
|
|
|
- // Move up to token end:
|
|
|
- int tokenEndIndex = token.getColumn() + token.getLength() - 1;
|
|
|
- columnIndex = appendPadding(builder, currentLine, columnIndex, tokenEndIndex);
|
|
|
-
|
|
|
- // Print token end if not already past it:
|
|
|
- if (columnIndex == tokenEndIndex) {
|
|
|
- builder.append(BAR);
|
|
|
- columnIndex++;
|
|
|
- }
|
|
|
- }
|
|
|
return builder.toString();
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * Appends the code line to the StringBuilder, applying the configuration of this TokenPrinter.
|
|
|
- * @param builder The StringBuilder to add code to
|
|
|
- * @param currentLine The line of code to append
|
|
|
- * @return a reference to that same StringBuilder
|
|
|
- */
|
|
|
- private static StringBuilder appendCodeLine(StringBuilder builder, String currentLine) {
|
|
|
- if (REPLACE_TABS) {
|
|
|
- currentLine = currentLine.replaceAll(TAB, TAB_REPLACEMENT);
|
|
|
- }
|
|
|
- return builder.append(currentLine);
|
|
|
- }
|
|
|
+ private static List<LineData> getLineData(List<Token> fileTokens, File root, Optional<String> suffix) {
|
|
|
+ // We expect that all fileTokens share the same Token.file!
|
|
|
|
|
|
- /**
|
|
|
- * Appends whitespace padding to the given StringBuilder in order to reach the targetPosition. Note that <b>the indices
|
|
|
- * are 1-based</b>, whereas the positions in currentLine are 0-based. The convention that lines start at position 1
|
|
|
- * comes from the frontends, specifically the Java frontend.
|
|
|
- * @param builder The StringBuilder to add padding to
|
|
|
- * @param currentLine The current line in the code file, indicating where it containes tab characters
|
|
|
- * @param currentPosition The (1-based) index of the last character within the currentLine that is labeled, or 1.
|
|
|
- * @param targetPosition The (1-based) index of the next character within the currentLine that should be labeled
|
|
|
- * @return the new position, which is the targetPosition or the end of the line.
|
|
|
- */
|
|
|
- private static int appendPadding(StringBuilder builder, String currentLine, int currentPosition, int targetPosition) {
|
|
|
- targetPosition = Math.min(targetPosition, currentLine.length());
|
|
|
- if (currentPosition >= targetPosition) {
|
|
|
- return currentPosition;
|
|
|
+ String fileName = fileTokens.get(0).getFile();
|
|
|
+ // handle 'files as submissions' mode
|
|
|
+ File file = fileName.isEmpty() ? root : new File(root, fileName);
|
|
|
+ if (suffix.isPresent()) {
|
|
|
+ file = new File(file.getPath() + suffix.get());
|
|
|
}
|
|
|
|
|
|
- // The replacement operation preserves TAB characters, which is essential for correct alignment
|
|
|
- String padding = currentLine.substring(currentPosition - 1, targetPosition - 1).replaceAll(TAB, REPLACE_TABS ? TAB_REPLACEMENT : TAB)
|
|
|
- .replaceAll(NON_WHITESPACE, SPACE);
|
|
|
- builder.append(padding);
|
|
|
-
|
|
|
- return targetPosition;
|
|
|
- }
|
|
|
+ // Sort tokens by file and line -> tokens can be processed without any further checks
|
|
|
+ List<String> lines = linesFromFile(file);
|
|
|
|
|
|
- private static StringBuilder appendCodeLinePrefix(StringBuilder builder, int index) {
|
|
|
- builder.append(System.lineSeparator()).append(index);
|
|
|
+ Map<Integer, List<Token>> lineNumbersToTokens = fileTokens.stream().collect(Collectors.groupingBy(Token::getLine));
|
|
|
|
|
|
- // Code line should be padded up to at least one tab length, and no less than MIN_PADDING
|
|
|
- int paddingLength = Math.max(TAB_LENGTH - digitCount(index), MIN_PADDING);
|
|
|
- return builder.append(SPACE.repeat(paddingLength));
|
|
|
+ // create LineData for each line -- 1-based line index
|
|
|
+ Stream<Integer> lineNumbers = PRINT_EMPTY_LINES ? IntStream.range(1, lines.size() + 1).boxed() : lineNumbersToTokens.keySet().stream();
|
|
|
+ return lineNumbers.map(lineIndex -> new LineData(lineIndex, lines.get(lineIndex - 1), lineNumbersToTokens.getOrDefault(lineIndex, List.of())))
|
|
|
+ .toList();
|
|
|
}
|
|
|
|
|
|
- private static StringBuilder appendTokenLinePrefix(StringBuilder builder, int index) {
|
|
|
- int paddingLength = Math.max(digitCount(index) + MIN_PADDING, TAB_LENGTH);
|
|
|
- String padding = SPACE.repeat(paddingLength);
|
|
|
- return builder.append(System.lineSeparator()).append(padding);
|
|
|
+ private static Map<String, List<Token>> groupTokensByFile(TokenList tokenList) {
|
|
|
+ return tokenList.allTokens().stream().collect(Collectors.groupingBy(Token::getFile));
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -205,13 +143,6 @@ public final class TokenPrinter {
|
|
|
return token.getLength() <= 1 && INDICATE_TINY_TOKEN ? token.toString().toLowerCase() : token.toString();
|
|
|
}
|
|
|
|
|
|
- /**
|
|
|
- * Turns a list of files into a map which maps the file names to a list of the contained lines of that file.
|
|
|
- */
|
|
|
- private static Map<String, List<String>> readFiles(Collection<File> allFiles) {
|
|
|
- return allFiles.stream().collect(toMap(File::getName, TokenPrinter::linesFromFile));
|
|
|
- }
|
|
|
-
|
|
|
/**
|
|
|
* Parses a file and returns a list of the contained lines.
|
|
|
*/
|
|
|
@@ -227,14 +158,179 @@ public final class TokenPrinter {
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
- * Returns the number of digits (including a minus) of the given number.
|
|
|
+ * This contains all data concerning a line of code in a file and the tokens found in that line.
|
|
|
+ * @param lineNumber the line number inside the file
|
|
|
+ * @param text the code line
|
|
|
+ * @param tokens the tokens found in the code line
|
|
|
*/
|
|
|
- private static int digitCount(int number) {
|
|
|
- if (number == 0) {
|
|
|
- return 1;
|
|
|
+ private record LineData(Integer lineNumber, String text, List<Token> tokens) {
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * A proxy for the StringBuilder that keeps track of the position inside the output.
|
|
|
+ */
|
|
|
+ private static class PrinterOutputBuilder {
|
|
|
+ public static final String LINE_SEPARATOR = System.lineSeparator();
|
|
|
+ private final StringBuilder builder = new StringBuilder();
|
|
|
+ private int columnIndex = 1;
|
|
|
+ private int lineNumber;
|
|
|
+ private int trailingLineSeparators = 0;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Returns the number of digits (including a minus) of the given number.
|
|
|
+ */
|
|
|
+ private static int digitCount(int number) {
|
|
|
+ if (number == 0) {
|
|
|
+ return 1;
|
|
|
+ }
|
|
|
+ int minusLength = number < 0 ? 1 : 0;
|
|
|
+ // The 'log10' variant is supposedly faster than the 'toString' variant.
|
|
|
+ return (int) Math.log10(Math.abs(number)) + minusLength + 1;
|
|
|
+ }
|
|
|
+
|
|
|
+ private void resetLinePosition() {
|
|
|
+ columnIndex = 1;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Appends the given string to the output
|
|
|
+ * @param str the string to append
|
|
|
+ * @return this
|
|
|
+ */
|
|
|
+ public PrinterOutputBuilder append(String str) {
|
|
|
+ // Avoid too many blank lines
|
|
|
+ trailingLineSeparators = str.equals(LINE_SEPARATOR) ? trailingLineSeparators + 1 : 0;
|
|
|
+ if (trailingLineSeparators >= 3)
|
|
|
+ return this;
|
|
|
+
|
|
|
+ builder.append(str);
|
|
|
+ columnIndex += str.length();
|
|
|
+ return this;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Appends the given integer to the output.
|
|
|
+ * @param i the integer to append
|
|
|
+ * @return this
|
|
|
+ */
|
|
|
+ public PrinterOutputBuilder append(int i) {
|
|
|
+ return append(Integer.toString(i));
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * In SPACIOUS mode, appends an empty line before the next code line.
|
|
|
+ * @return a reference to this
|
|
|
+ */
|
|
|
+ private PrinterOutputBuilder appendTokenLineSuffix() {
|
|
|
+ return SPACIOUS ? advanceToNextLine() : this;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Appends the code line to the StringBuilder, applying the configuration of this TokenPrinter.
|
|
|
+ * @param currentLine The line of code to append
|
|
|
+ * @return a reference to this
|
|
|
+ */
|
|
|
+ private PrinterOutputBuilder appendCodeLine(String currentLine) {
|
|
|
+ // Prefix
|
|
|
+ advanceToNextLine().append(lineNumber);
|
|
|
+
|
|
|
+ // Code line should be padded up to at least one tab length, and no less than MIN_PADDING
|
|
|
+ int paddingLength = Math.max(TAB_LENGTH - digitCount(lineNumber), MIN_PADDING);
|
|
|
+ appendPadding(paddingLength);
|
|
|
+
|
|
|
+ // Code line
|
|
|
+ if (REPLACE_TABS) {
|
|
|
+ currentLine = currentLine.replaceAll(TAB, TAB_REPLACEMENT);
|
|
|
+ }
|
|
|
+
|
|
|
+ return append(currentLine);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Appends whitespace padding to the given StringBuilder in order to reach the targetPosition. Note that <b>the indices
|
|
|
+ * are 1-based</b>, whereas the positions in currentLine are 0-based. The convention that lines start at position 1
|
|
|
+ * comes from the frontends, specifically the Java frontend.
|
|
|
+ * @param currentLine The current line in the code file, indicating where it containes tab characters
|
|
|
+ * @param targetPosition The (1-based) index of the next character within the currentLine that should be labeled
|
|
|
+ * @param breakLine If true, a lineSeparator will be added if the currentPosition is past targetPosition.
|
|
|
+ * @return the new position, which is the targetPosition or the end of the line.
|
|
|
+ */
|
|
|
+ PrinterOutputBuilder advanceToTokenPosition(String currentLine, int targetPosition, boolean breakLine) {
|
|
|
+ targetPosition = Math.min(targetPosition, currentLine.length());
|
|
|
+ if (!positionBeforeOrEqualTo(targetPosition)) {
|
|
|
+ if (!breakLine) {
|
|
|
+ // we are past targetPosition, do nothing
|
|
|
+ return this;
|
|
|
+ }
|
|
|
+
|
|
|
+ // new line
|
|
|
+ appendTokenLinePrefix();
|
|
|
+ if (!positionBeforeOrEqualTo(targetPosition)) {
|
|
|
+ // still past targetPosition -> negative targetPosition
|
|
|
+ return this;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // The replacement operation preserves TAB characters, which is essential for correct alignment
|
|
|
+ String padding = currentLine.substring(columnIndex - 1, targetPosition - 1)
|
|
|
+ // TAB is a string, so use replace
|
|
|
+ .replace(TAB, REPLACE_TABS ? TAB_REPLACEMENT : TAB)
|
|
|
+ // NON_WHITESPACE is a pattern, so use replaceAll
|
|
|
+ .replaceAll(NON_WHITESPACE, SPACE);
|
|
|
+ append(padding);
|
|
|
+
|
|
|
+ return this;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Returns true if the current position is before or at the target position.
|
|
|
+ * @param targetPosition the target position
|
|
|
+ * @return true if before or at target position
|
|
|
+ */
|
|
|
+ private boolean positionBeforeOrEqualTo(int targetPosition) {
|
|
|
+ return columnIndex <= targetPosition;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Appends a newline, plus the amount of padding necessary to align tokens correctly.
|
|
|
+ * @return this
|
|
|
+ */
|
|
|
+ PrinterOutputBuilder appendTokenLinePrefix() {
|
|
|
+ int paddingLength = Math.max(digitCount(lineNumber) + MIN_PADDING, TAB_LENGTH);
|
|
|
+ advanceToNextLine().appendPadding(paddingLength);
|
|
|
+ resetLinePosition();
|
|
|
+ return this;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Appends the amount of padding given.
|
|
|
+ * @param paddingLength Length of padding to add
|
|
|
+ * @return this
|
|
|
+ */
|
|
|
+ private PrinterOutputBuilder appendPadding(int paddingLength) {
|
|
|
+ String padding = SPACE.repeat(paddingLength);
|
|
|
+ append(padding);
|
|
|
+ return this;
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Appends a newline.
|
|
|
+ * @return this
|
|
|
+ */
|
|
|
+ private PrinterOutputBuilder advanceToNextLine() {
|
|
|
+ append(LINE_SEPARATOR);
|
|
|
+ resetLinePosition();
|
|
|
+ return this;
|
|
|
+ }
|
|
|
+
|
|
|
+ public void setLine(int lineNumber) {
|
|
|
+ this.lineNumber = lineNumber;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public String toString() {
|
|
|
+ return builder.toString();
|
|
|
}
|
|
|
- int minusLength = number < 0 ? 1 : 0;
|
|
|
- // The 'log10' variant is supposedly faster than the 'toString' variant.
|
|
|
- return (int) Math.log10(Math.abs(number)) + minusLength + 1;
|
|
|
}
|
|
|
}
|