package de.jplag;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Utility class for printing JPlag tokens from a submission. Each line of code is printed starting with the line
* number. Under these lines the tokens are annotated in the format |TOKEN|. The first vertical line marks
* the token start, while the last vertical line marks the token end. Tokens that are shorter than the name do not end
* with a vertical line, e.g. |TOKEN. Tokens with length 1 or 0 are printed in lower case, e.g.
* |token.
* @author Timur Saglam
*/
public final class TokenPrinter {
private static final Logger logger = LoggerFactory.getLogger(TokenPrinter.class);
// Representation:
private static final String BAR = "|";
private static final String TAB = "\t";
private static final String SPACE = " ";
private static final String NON_WHITESPACE = "\\S";
private static final int MIN_PADDING = 1;
private static final int TAB_LENGTH = 8;
private static final String TAB_REPLACEMENT = SPACE.repeat(TAB_LENGTH); // might depend on files
// Configuration:
private static final boolean INDICATE_TINY_TOKEN = true; // print token with length <= 1 in lowercase
private static final boolean REPLACE_TABS = false;
private static final boolean PRINT_EMPTY_LINES = true; // print code lines with no tokens
private static final boolean SPACIOUS = true; // print empty line after last token of a line
private TokenPrinter() {
// Utility class, no public constructor.
}
/**
* Creates a string representation of a set of files line by line and adds the tokens under the lines.
* @param tokens is the list of tokens parsed from the files.
* @param rootDirectory is the common rootDirectory of the files.
* @return the string representation.
*/
public static String printTokens(List tokens, File rootDirectory) {
return printTokens(tokens, rootDirectory, Optional.empty());
}
/**
* Creates a string representation of a collection of files line by line and adds the tokens under the lines.
* @param tokenList is the list of tokens parsed from the files.
* @param rootDirectory is the common directory of the files.
* @param suffix is the optional view file suffix.
* @return the string representation.
*/
public static String printTokens(List tokenList, File rootDirectory, Optional suffix) {
PrinterOutputBuilder builder = new PrinterOutputBuilder();
Map> fileToTokens = groupTokensByFile(tokenList);
fileToTokens.forEach((String fileName, List fileTokens) -> {
builder.append(fileName);
List lineDatas = getLineData(fileTokens, rootDirectory, suffix);
lineDatas.forEach(lineData -> {
builder.setLine(lineData.lineNumber());
// Print (prefix and) code line
String currentLine = lineData.text();
builder.appendCodeLine(currentLine);
List tokens = lineData.tokens();
if (tokens.isEmpty()) {
return;
}
builder.appendTokenLinePrefix();
// Print tokens
for (Token token : tokens) {
// Move to token index, possibly adding a new line:
builder.advanceToTokenPosition(currentLine, token.getColumn(), true);
// Print the actual token:
String stringRepresentation = getStringRepresentation(token);
builder.append(BAR).append(stringRepresentation);
// Move up to token end:
int tokenEndIndex = token.getColumn() + token.getLength() - 1;
builder.advanceToTokenPosition(currentLine, tokenEndIndex, false);
// Print token end if not already past it:
if (builder.positionBeforeOrEqualTo(tokenEndIndex)) {
builder.append(BAR);
}
}
builder.appendTokenLineSuffix();
builder.advanceToNextLine();
});
builder.advanceToNextLine();
});
return builder.toString();
}
private static List getLineData(List fileTokens, File root, Optional suffix) {
// We expect that all fileTokens share the same Token.file!
String fileName = fileTokens.get(0).getFile();
// handle 'files as submissions' mode
File file = fileName.isEmpty() ? root : new File(root, fileName);
if (suffix.isPresent()) {
file = new File(file.getPath() + suffix.get());
}
// Sort tokens by file and line -> tokens can be processed without any further checks
List lines = linesFromFile(file);
int currentLine = Token.NO_VALUE;
Map> lineNumbersToTokens = new HashMap<>(fileTokens.size());
for (Token token : fileTokens) {
if (token.getLine() != Token.NO_VALUE) {
currentLine = token.getLine();
}
int line = token.getType() == TokenConstants.FILE_END ? lines.size() : currentLine;
List tokens = lineNumbersToTokens.containsKey(line) ? lineNumbersToTokens.get(line) : new ArrayList<>();
tokens.add(token);
lineNumbersToTokens.put(line, tokens);
}
// create LineData for each line -- 1-based line index
Stream lineNumbers = PRINT_EMPTY_LINES ? IntStream.range(1, lines.size() + 1).boxed() : lineNumbersToTokens.keySet().stream();
return lineNumbers.map(lineIndex -> new LineData(lineIndex, lines.get(lineIndex - 1), lineNumbersToTokens.getOrDefault(lineIndex, List.of())))
.toList();
}
private static Map> groupTokensByFile(List tokens) {
return tokens.stream().collect(Collectors.groupingBy(Token::getFile));
}
/**
* Determines the string representation of the token.
*/
private static String getStringRepresentation(Token token) {
return token.getLength() <= 1 && INDICATE_TINY_TOKEN ? token.toString().toLowerCase() : token.toString();
}
/**
* Parses a file and returns a list of the contained lines.
*/
private static List linesFromFile(File file) {
try {
return Files.readAllLines(file.toPath());
} catch (NoSuchFileException exception) {
logger.error("File does not exist, thus no tokens are printed: " + file.getAbsolutePath());
} catch (IOException exception) {
logger.error("Cannot read " + file.getAbsolutePath() + ":", exception);
}
return Collections.emptyList();
}
/**
* This contains all data concerning a line of code in a file and the tokens found in that line.
* @param lineNumber the line number inside the file
* @param text the code line
* @param tokens the tokens found in the code line
*/
private record LineData(Integer lineNumber, String text, List tokens) {
}
/**
* A proxy for the StringBuilder that keeps track of the position inside the output.
*/
private static class PrinterOutputBuilder {
public static final String LINE_SEPARATOR = System.lineSeparator();
private final StringBuilder builder = new StringBuilder();
private int columnIndex = 1;
private int lineNumber;
private int trailingLineSeparators = 0;
/**
* Returns the number of digits (including a minus) of the given number.
*/
private static int digitCount(int number) {
if (number == 0) {
return 1;
}
int minusLength = number < 0 ? 1 : 0;
// The 'log10' variant is supposedly faster than the 'toString' variant.
return (int) Math.log10(Math.abs(number)) + minusLength + 1;
}
private void resetLinePosition() {
columnIndex = 1;
}
/**
* Appends the given string to the output
* @param str the string to append
* @return this
*/
public PrinterOutputBuilder append(String str) {
// Avoid too many blank lines
trailingLineSeparators = str.equals(LINE_SEPARATOR) ? trailingLineSeparators + 1 : 0;
if (trailingLineSeparators >= 3)
return this;
builder.append(str);
columnIndex += str.length();
return this;
}
/**
* Appends the given integer to the output.
* @param i the integer to append
* @return this
*/
public PrinterOutputBuilder append(int i) {
return append(Integer.toString(i));
}
/**
* In SPACIOUS mode, appends an empty line before the next code line.
* @return a reference to this
*/
private PrinterOutputBuilder appendTokenLineSuffix() {
return SPACIOUS ? advanceToNextLine() : this;
}
/**
* Appends the code line to the StringBuilder, applying the configuration of this TokenPrinter.
* @param currentLine The line of code to append
* @return a reference to this
*/
private PrinterOutputBuilder appendCodeLine(String currentLine) {
// Prefix
advanceToNextLine().append(lineNumber);
// Code line should be padded up to at least one tab length, and no less than MIN_PADDING
int paddingLength = Math.max(TAB_LENGTH - digitCount(lineNumber), MIN_PADDING);
appendPadding(paddingLength);
// Code line
if (REPLACE_TABS) {
currentLine = currentLine.replaceAll(TAB, TAB_REPLACEMENT);
}
return append(currentLine);
}
/**
* Appends whitespace padding to the given StringBuilder in order to reach the targetPosition. Note that the indices
* are 1-based, whereas the positions in currentLine are 0-based. The convention that lines start at position 1
* comes from the frontends, specifically the Java frontend.
* @param currentLine The current line in the code file, indicating where it containes tab characters
* @param targetPosition The (1-based) index of the next character within the currentLine that should be labeled
* @param breakLine If true, a lineSeparator will be added if the currentPosition is past targetPosition.
* @return the new position, which is the targetPosition or the end of the line.
*/
PrinterOutputBuilder advanceToTokenPosition(String currentLine, int targetPosition, boolean breakLine) {
targetPosition = Math.min(targetPosition, currentLine.length());
if (!positionBeforeOrEqualTo(targetPosition)) {
if (!breakLine) {
// we are past targetPosition, do nothing
return this;
}
// new line
appendTokenLinePrefix();
if (!positionBeforeOrEqualTo(targetPosition)) {
// still past targetPosition -> negative targetPosition
return this;
}
}
// The replacement operation preserves TAB characters, which is essential for correct alignment
String padding = currentLine.substring(columnIndex - 1, targetPosition - 1)
// TAB is a string, so use replace
.replace(TAB, REPLACE_TABS ? TAB_REPLACEMENT : TAB)
// NON_WHITESPACE is a pattern, so use replaceAll
.replaceAll(NON_WHITESPACE, SPACE);
append(padding);
return this;
}
/**
* Returns true if the current position is before or at the target position.
* @param targetPosition the target position
* @return true if before or at target position
*/
private boolean positionBeforeOrEqualTo(int targetPosition) {
return columnIndex <= targetPosition;
}
/**
* Appends a newline, plus the amount of padding necessary to align tokens correctly.
* @return this
*/
PrinterOutputBuilder appendTokenLinePrefix() {
int paddingLength = Math.max(digitCount(lineNumber) + MIN_PADDING, TAB_LENGTH);
advanceToNextLine().appendPadding(paddingLength);
resetLinePosition();
return this;
}
/**
* Appends the amount of padding given.
* @param paddingLength Length of padding to add
* @return this
*/
private PrinterOutputBuilder appendPadding(int paddingLength) {
String padding = SPACE.repeat(paddingLength);
append(padding);
return this;
}
/**
* Appends a newline.
* @return this
*/
private PrinterOutputBuilder advanceToNextLine() {
append(LINE_SEPARATOR);
resetLinePosition();
return this;
}
public void setLine(int lineNumber) {
this.lineNumber = lineNumber;
}
@Override
public String toString() {
return builder.toString();
}
}
}