package de.jplag;
import static java.util.stream.Collectors.toMap;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Utility class for printing JPlag tokens from a submission. Each line of code is printed starting with the line
* number. Under these lines the tokens are annotated in the format |TOKEN|. The first vertical line marks
* the token start, while the last vertical line marks the token end. Tokens that are shorter than the name do not end
* with a vertical line, e.g. |TOKEN. Tokens with length 1 or 0 are printed in lower case, e.g.
* |token.
* @author Timur Saglam
*/
public final class TokenPrinter {
public static final String NON_WHITESPACE = "\\S";
public static final int MIN_PADDING = 1;
private static final Logger logger = LoggerFactory.getLogger(TokenPrinter.class);
// Representation:
private static final String BAR = "|";
private static final String TAB = "\t";
private static final String SPACE = " ";
private static final int TAB_LENGTH = 8;
private static final String TAB_REPLACEMENT = SPACE.repeat(TAB_LENGTH); // might depend on files
// Configuration:
private static final boolean INDICATE_TINY_TOKEN = true; // print token with length <= 1 in lowercase
private static final boolean REPLACE_TABS = false;
private static final boolean PRINT_EMPTY_LINES = true; // print code lines with no tokens
private static final boolean SPACIOUS = true; // print empty line after last token of a line
private TokenPrinter() {
// Utility class, no public constructor.
}
/**
* Creates a string representation of a set of files line by line and adds the tokens under the lines.
* @param tokens is the set of tokens parsed from the files.
* @param directory is the common directory of the files.
* @param fileNames is a collection of the file names.
* @param suffix is the optional view file suffix.
* @return the string representation.
*/
public static String printTokens(TokenList tokens, File directory, Collection fileNames, Optional suffix) {
Collection files = fileNames.stream().map(name -> new File(directory, name)).toList();
return printTokens(tokens, files, directory, suffix);
}
/**
* Creates a string representation of a set of files line by line and adds the tokens under the lines.
* @param tokens is the set of tokens parsed from the files.
* @param directory is the common directory of the files.
* @param fileNames is a collection of the file names.
* @return the string representation.
*/
public static String printTokens(TokenList tokens, File directory, Collection fileNames) {
Collection files = fileNames.stream().map(name -> new File(directory, name)).toList();
return printTokens(tokens, files, directory, Optional.empty());
}
/**
* Creates a string representation of a collection of files line by line and adds the tokens under the lines.
* @param tokens is the set of tokens parsed from the files.
* @param files are the parsed files.
* @return the string representation.
*/
public static String printTokens(TokenList tokens, Collection files, File root) {
return printTokens(tokens, files, root, Optional.empty());
}
/**
* Creates a string representation of a collection of files line by line and adds the tokens under the lines.
* @param tokenList is the set of tokens parsed from the files.
* @param files are the parsed files.
* @param root is the common directory of the files.
* @param suffix is the optional view file suffix.
* @return the string representation.
*/
public static String printTokens(TokenList tokenList, Collection files, File root, Optional suffix) {
PrinterOutputBuilder builder = new PrinterOutputBuilder();
Map> fileToLineData = getLineData(files, tokenList, suffix, root);
fileToLineData.forEach((String fileName, List lineDatas) -> {
builder.append(fileName);
LineData.forEachApply(lineDatas, (String file, Integer lineNumber, String currentLine, List tokens) -> {
builder.setLine(lineNumber);
// Print (prefix and) code line
builder.appendCodeLine(currentLine);
if (tokens.isEmpty()) {
return;
}
builder.appendTokenLinePrefix();
// Print tokens
for (Token token : tokens) {
// Move to token index, possibly adding a new line:
builder.advanceToTokenPosition(currentLine, token.getColumn(), true);
// Print the actual token:
String stringRepresentation = getStringRepresentation(token);
builder.append(BAR).append(stringRepresentation);
// Move up to token end:
int tokenEndIndex = token.getColumn() + token.getLength() - 1;
builder.advanceToTokenPosition(currentLine, tokenEndIndex, false);
// Print token end if not already past it:
if (builder.positionBeforeOrEqualTo(tokenEndIndex)) {
builder.append(BAR);
}
}
builder.appendTokenLineSuffix();
});
builder.advanceToNextLine();
});
return builder.toString();
}
/**
* Determines the string representation of the token.
*/
private static String getStringRepresentation(Token token) {
return token.getLength() <= 1 && INDICATE_TINY_TOKEN ? token.toString().toLowerCase() : token.toString();
}
/**
* Turns a list of files into a map which maps the file names to a list of the contained lines of that file.
*/
private static Map> readFiles(Collection allFiles) {
return allFiles.stream().collect(toMap(File::getName, TokenPrinter::linesFromFile));
}
/**
* Turns a list of files and a list of tokens into a map of the file name to [a list of LineData objects for the lines
* of the file].
* @param allFiles a collection of files
* @param allTokens a TokenList containing all tokens of the files
* @param suffix the optional view file suffix
* @param root the root element of the submission
* @return map of file -> [LineData]
*/
private static Map> getLineData(Collection allFiles, TokenList allTokens, Optional suffix, File root) {
Map> filesToTokens = allTokens.stream().collect(Collectors.groupingBy(Token::getFile));
Map> filesToLines = readFiles(allFiles);
// handle 'files as submissions' mode
final String EMPTY_STRING = "";
if (filesToTokens.containsKey(EMPTY_STRING)) {
filesToTokens.put(root.getName(), filesToTokens.get(EMPTY_STRING));
filesToTokens.remove(EMPTY_STRING);
}
// Sort tokens by file and line -> tokens can be processed without any further checks
Function> fileToLineDataList = file -> {
String fileName = file.getName();
String tokenFileName = restoreTokenFileName(file.getName(), suffix);
List lines = filesToLines.get(fileName);
Map> lineNumbersToTokens = filesToTokens.get(tokenFileName).stream().collect(Collectors.groupingBy(Token::getLine));
// create LineData for each line -- 1-based line index
return IntStream.range(1, lines.size() + 1).mapToObj(
lineIndex -> new LineData(fileName, lineIndex, lines.get(lineIndex - 1), lineNumbersToTokens.getOrDefault(lineIndex, List.of())))
.filter(lineData -> PRINT_EMPTY_LINES || !lineData.tokens.isEmpty()).toList();
};
return allFiles.stream().collect(toMap(File::getName, fileToLineDataList));
}
/**
* Returns the fileName with the given suffix cut off at the end, if any.
* @param fileName the file name with a suffix
* @param suffix the suffix to cut off
* @return the shortened file name
*/
private static String restoreTokenFileName(String fileName, Optional suffix) {
if (suffix.isEmpty() || !fileName.endsWith(suffix.get())) {
return fileName;
}
int newLength = fileName.length() - suffix.get().length();
return fileName.substring(0, newLength);
}
/**
* Parses a file and returns a list of the contained lines.
*/
private static List linesFromFile(File file) {
try {
return Files.readAllLines(file.toPath());
} catch (NoSuchFileException exception) {
logger.error("File does not exist, thus no tokens are printed: " + file.getAbsolutePath());
} catch (IOException exception) {
logger.error("Cannot read " + file.getAbsolutePath() + ":", exception);
}
return Collections.emptyList();
}
/**
* This contains all data concerning a line of code in a file and the tokens found in that line.
* @param file the file
* @param lineNumber the line number inside the file
* @param text the code line
* @param tokens the tokens found in the code line
*/
private record LineData(String file, Integer lineNumber, String text, List tokens) {
/**
* Applies the given LineDataConsumer to this LineData.
* @param consumer the LineDataConsumer
*/
public void apply(LineDataConsumer consumer) {
consumer.apply(file, lineNumber, text, tokens);
}
/**
* Convenience method to apply a LineDataConsumer to all lineDatas in the given collection.
* @param lineDatas the LineData objects to apply the consumer to
* @param consumer the LineDataConsumer to apply
*/
public static void forEachApply(Collection lineDatas, LineDataConsumer consumer) {
lineDatas.forEach(lineData -> lineData.apply(consumer));
}
/**
* The LineDataConsumer interface allows to declare variables for all fields of a LineData object in a functional
* interface, thus saving manually declaring variables for each one.
*/
@FunctionalInterface
interface LineDataConsumer {
void apply(String file, Integer lineNumber, String text, List tokens);
default void apply(LineData lineData) {
apply(lineData.file, lineData.lineNumber, lineData.text, lineData.tokens);
}
}
}
/**
* A proxy for the StringBuilder that keeps track of the position inside the output.
*/
private static class PrinterOutputBuilder {
public static final String LINE_SEPARATOR = System.lineSeparator();
private final StringBuilder builder = new StringBuilder();
private int columnIndex = 1;
private int lineNumber;
private int trailingLineSeparators = 0;
/**
* Returns the number of digits (including a minus) of the given number.
*/
private static int digitCount(int number) {
if (number == 0) {
return 1;
}
int minusLength = number < 0 ? 1 : 0;
// The 'log10' variant is supposedly faster than the 'toString' variant.
return (int) Math.log10(Math.abs(number)) + minusLength + 1;
}
private void resetLinePosition() {
columnIndex = 1;
}
/**
* Appends the given string to the output
* @param str the string to append
* @return this
*/
public PrinterOutputBuilder append(String str) {
// Avoid too many blank lines
trailingLineSeparators = str.equals(LINE_SEPARATOR) ? trailingLineSeparators + 1 : 0;
if (trailingLineSeparators >= 2)
return this;
builder.append(str);
columnIndex += str.length();
return this;
}
/**
* Appends the given integer to the output.
* @param i the integer to append
* @return this
*/
public PrinterOutputBuilder append(int i) {
return append(Integer.toString(i));
}
/**
* In SPACIOUS mode, appends an empty line before the next code line.
* @return a reference to this
*/
private PrinterOutputBuilder appendTokenLineSuffix() {
return SPACIOUS ? advanceToNextLine() : this;
}
/**
* Appends the code line to the StringBuilder, applying the configuration of this TokenPrinter.
* @param currentLine The line of code to append
* @return a reference to this
*/
private PrinterOutputBuilder appendCodeLine(String currentLine) {
// Prefix
advanceToNextLine().append(lineNumber);
// Code line should be padded up to at least one tab length, and no less than MIN_PADDING
int paddingLength = Math.max(TAB_LENGTH - digitCount(lineNumber), MIN_PADDING);
appendPadding(paddingLength);
// Code line
if (REPLACE_TABS) {
currentLine = currentLine.replaceAll(TAB, TAB_REPLACEMENT);
}
return append(currentLine);
}
/**
* Appends whitespace padding to the given StringBuilder in order to reach the targetPosition. Note that the indices
* are 1-based, whereas the positions in currentLine are 0-based. The convention that lines start at position 1
* comes from the frontends, specifically the Java frontend.
* @param currentLine The current line in the code file, indicating where it containes tab characters
* @param targetPosition The (1-based) index of the next character within the currentLine that should be labeled
* @param breakLine If true, a lineSeparator will be added if the currentPosition is past targetPosition.
* @return the new position, which is the targetPosition or the end of the line.
*/
PrinterOutputBuilder advanceToTokenPosition(String currentLine, int targetPosition, boolean breakLine) {
targetPosition = Math.min(targetPosition, currentLine.length());
if (positionBeforeOrEqualTo(targetPosition)) {
if (!breakLine) {
// we are past targetPosition, do nothing
return this;
}
// new line
appendTokenLinePrefix();
if (positionBeforeOrEqualTo(targetPosition)) {
// still past targetPosition -> negative targetPosition
return this;
}
}
// The replacement operation preserves TAB characters, which is essential for correct alignment
String padding = currentLine.substring(columnIndex - 1, targetPosition - 1).replace(TAB, REPLACE_TABS ? TAB_REPLACEMENT : TAB)
.replace(NON_WHITESPACE, SPACE);
append(padding);
return this;
}
/**
* Returns true if the current position is before or at the target position.
* @param targetPosition the target position
* @return true if before or at target position
*/
private boolean positionBeforeOrEqualTo(int targetPosition) {
return columnIndex > targetPosition;
}
/**
* Appends a newline, plus the amount of padding necessary to align tokens correctly.
* @return this
*/
PrinterOutputBuilder appendTokenLinePrefix() {
int paddingLength = Math.max(digitCount(lineNumber) + MIN_PADDING, TAB_LENGTH);
return advanceToNextLine().appendPadding(paddingLength);
}
/**
* Appends the amount of padding given.
* @param paddingLength Length of padding to add
* @return this
*/
private PrinterOutputBuilder appendPadding(int paddingLength) {
String padding = SPACE.repeat(paddingLength);
append(padding);
return this;
}
/**
* Appends a newline.
* @return this
*/
private PrinterOutputBuilder advanceToNextLine() {
append(LINE_SEPARATOR);
resetLinePosition();
return this;
}
public void setLine(int lineNumber) {
this.lineNumber = lineNumber;
}
@Override
public String toString() {
return builder.toString();
}
}
}