package de.jplag; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Utility class for printing JPlag tokens from a submission. Each line of code is printed starting with the line * number. Under these lines the tokens are annotated in the format |TOKEN|. The first vertical line marks * the token start, while the last vertical line marks the token end. Tokens that are shorter than the name do not end * with a vertical line, e.g. |TOKEN. Tokens with length 1 or 0 are printed in lower case, e.g. * |token. * @author Timur Saglam */ public final class TokenPrinter { private static final Logger logger = LoggerFactory.getLogger(TokenPrinter.class); // Representation: private static final String BAR = "|"; private static final String TAB = "\t"; private static final String SPACE = " "; private static final String NON_WHITESPACE = "\\S"; private static final int MIN_PADDING = 1; private static final int TAB_LENGTH = 8; private static final String TAB_REPLACEMENT = SPACE.repeat(TAB_LENGTH); // might depend on files // Configuration: private static final boolean INDICATE_TINY_TOKEN = true; // print token with length <= 1 in lowercase private static final boolean REPLACE_TABS = false; private static final boolean PRINT_EMPTY_LINES = true; // print code lines with no tokens private static final boolean SPACIOUS = true; // print empty line after last token of a line private TokenPrinter() { // Utility class, no public constructor. } /** * Creates a string representation of a set of files line by line and adds the tokens under the lines. * @param tokens is the list of tokens parsed from the files. * @param rootDirectory is the common rootDirectory of the files. * @return the string representation. */ public static String printTokens(List tokens, File rootDirectory) { return printTokens(tokens, rootDirectory, Optional.empty()); } /** * Creates a string representation of a collection of files line by line and adds the tokens under the lines. * @param tokenList is the list of tokens parsed from the files. * @param rootDirectory is the common directory of the files. * @param suffix is the optional view file suffix. * @return the string representation. */ public static String printTokens(List tokenList, File rootDirectory, Optional suffix) { PrinterOutputBuilder builder = new PrinterOutputBuilder(); Map> fileToTokens = groupTokensByFile(tokenList); fileToTokens.forEach((String fileName, List fileTokens) -> { builder.append(fileName); List lineDatas = getLineData(fileTokens, rootDirectory, suffix); lineDatas.forEach(lineData -> { builder.setLine(lineData.lineNumber()); // Print (prefix and) code line String currentLine = lineData.text(); builder.appendCodeLine(currentLine); List tokens = lineData.tokens(); if (tokens.isEmpty()) { return; } builder.appendTokenLinePrefix(); // Print tokens for (Token token : tokens) { // Move to token index, possibly adding a new line: builder.advanceToTokenPosition(currentLine, token.getColumn(), true); // Print the actual token: String stringRepresentation = getStringRepresentation(token); builder.append(BAR).append(stringRepresentation); // Move up to token end: int tokenEndIndex = token.getColumn() + token.getLength() - 1; builder.advanceToTokenPosition(currentLine, tokenEndIndex, false); // Print token end if not already past it: if (builder.positionBeforeOrEqualTo(tokenEndIndex)) { builder.append(BAR); } } builder.appendTokenLineSuffix(); builder.advanceToNextLine(); }); builder.advanceToNextLine(); }); return builder.toString(); } private static List getLineData(List fileTokens, File root, Optional suffix) { // We expect that all fileTokens share the same Token.file! String fileName = fileTokens.get(0).getFile(); // handle 'files as submissions' mode File file = fileName.isEmpty() ? root : new File(root, fileName); if (suffix.isPresent()) { file = new File(file.getPath() + suffix.get()); } // Sort tokens by file and line -> tokens can be processed without any further checks List lines = linesFromFile(file); int currentLine = Token.NO_VALUE; Map> lineNumbersToTokens = new HashMap<>(fileTokens.size()); for (Token token : fileTokens) { if (token.getLine() != Token.NO_VALUE) { currentLine = token.getLine(); } int line = token.getType() == TokenConstants.FILE_END ? lines.size() : currentLine; List tokens = lineNumbersToTokens.containsKey(line) ? lineNumbersToTokens.get(line) : new ArrayList<>(); tokens.add(token); lineNumbersToTokens.put(line, tokens); } // create LineData for each line -- 1-based line index Stream lineNumbers = PRINT_EMPTY_LINES ? IntStream.range(1, lines.size() + 1).boxed() : lineNumbersToTokens.keySet().stream(); return lineNumbers.map(lineIndex -> new LineData(lineIndex, lines.get(lineIndex - 1), lineNumbersToTokens.getOrDefault(lineIndex, List.of()))) .toList(); } private static Map> groupTokensByFile(List tokens) { return tokens.stream().collect(Collectors.groupingBy(Token::getFile)); } /** * Determines the string representation of the token. */ private static String getStringRepresentation(Token token) { return token.getLength() <= 1 && INDICATE_TINY_TOKEN ? token.toString().toLowerCase() : token.toString(); } /** * Parses a file and returns a list of the contained lines. */ private static List linesFromFile(File file) { try { return Files.readAllLines(file.toPath()); } catch (NoSuchFileException exception) { logger.error("File does not exist, thus no tokens are printed: " + file.getAbsolutePath()); } catch (IOException exception) { logger.error("Cannot read " + file.getAbsolutePath() + ":", exception); } return Collections.emptyList(); } /** * This contains all data concerning a line of code in a file and the tokens found in that line. * @param lineNumber the line number inside the file * @param text the code line * @param tokens the tokens found in the code line */ private record LineData(Integer lineNumber, String text, List tokens) { } /** * A proxy for the StringBuilder that keeps track of the position inside the output. */ private static class PrinterOutputBuilder { public static final String LINE_SEPARATOR = System.lineSeparator(); private final StringBuilder builder = new StringBuilder(); private int columnIndex = 1; private int lineNumber; private int trailingLineSeparators = 0; /** * Returns the number of digits (including a minus) of the given number. */ private static int digitCount(int number) { if (number == 0) { return 1; } int minusLength = number < 0 ? 1 : 0; // The 'log10' variant is supposedly faster than the 'toString' variant. return (int) Math.log10(Math.abs(number)) + minusLength + 1; } private void resetLinePosition() { columnIndex = 1; } /** * Appends the given string to the output * @param str the string to append * @return this */ public PrinterOutputBuilder append(String str) { // Avoid too many blank lines trailingLineSeparators = str.equals(LINE_SEPARATOR) ? trailingLineSeparators + 1 : 0; if (trailingLineSeparators >= 3) return this; builder.append(str); columnIndex += str.length(); return this; } /** * Appends the given integer to the output. * @param i the integer to append * @return this */ public PrinterOutputBuilder append(int i) { return append(Integer.toString(i)); } /** * In SPACIOUS mode, appends an empty line before the next code line. * @return a reference to this */ private PrinterOutputBuilder appendTokenLineSuffix() { return SPACIOUS ? advanceToNextLine() : this; } /** * Appends the code line to the StringBuilder, applying the configuration of this TokenPrinter. * @param currentLine The line of code to append * @return a reference to this */ private PrinterOutputBuilder appendCodeLine(String currentLine) { // Prefix advanceToNextLine().append(lineNumber); // Code line should be padded up to at least one tab length, and no less than MIN_PADDING int paddingLength = Math.max(TAB_LENGTH - digitCount(lineNumber), MIN_PADDING); appendPadding(paddingLength); // Code line if (REPLACE_TABS) { currentLine = currentLine.replaceAll(TAB, TAB_REPLACEMENT); } return append(currentLine); } /** * Appends whitespace padding to the given StringBuilder in order to reach the targetPosition. Note that the indices * are 1-based, whereas the positions in currentLine are 0-based. The convention that lines start at position 1 * comes from the frontends, specifically the Java frontend. * @param currentLine The current line in the code file, indicating where it containes tab characters * @param targetPosition The (1-based) index of the next character within the currentLine that should be labeled * @param breakLine If true, a lineSeparator will be added if the currentPosition is past targetPosition. * @return the new position, which is the targetPosition or the end of the line. */ PrinterOutputBuilder advanceToTokenPosition(String currentLine, int targetPosition, boolean breakLine) { targetPosition = Math.min(targetPosition, currentLine.length()); if (!positionBeforeOrEqualTo(targetPosition)) { if (!breakLine) { // we are past targetPosition, do nothing return this; } // new line appendTokenLinePrefix(); if (!positionBeforeOrEqualTo(targetPosition)) { // still past targetPosition -> negative targetPosition return this; } } // The replacement operation preserves TAB characters, which is essential for correct alignment String padding = currentLine.substring(columnIndex - 1, targetPosition - 1) // TAB is a string, so use replace .replace(TAB, REPLACE_TABS ? TAB_REPLACEMENT : TAB) // NON_WHITESPACE is a pattern, so use replaceAll .replaceAll(NON_WHITESPACE, SPACE); append(padding); return this; } /** * Returns true if the current position is before or at the target position. * @param targetPosition the target position * @return true if before or at target position */ private boolean positionBeforeOrEqualTo(int targetPosition) { return columnIndex <= targetPosition; } /** * Appends a newline, plus the amount of padding necessary to align tokens correctly. * @return this */ PrinterOutputBuilder appendTokenLinePrefix() { int paddingLength = Math.max(digitCount(lineNumber) + MIN_PADDING, TAB_LENGTH); advanceToNextLine().appendPadding(paddingLength); resetLinePosition(); return this; } /** * Appends the amount of padding given. * @param paddingLength Length of padding to add * @return this */ private PrinterOutputBuilder appendPadding(int paddingLength) { String padding = SPACE.repeat(paddingLength); append(padding); return this; } /** * Appends a newline. * @return this */ private PrinterOutputBuilder advanceToNextLine() { append(LINE_SEPARATOR); resetLinePosition(); return this; } public void setLine(int lineNumber) { this.lineNumber = lineNumber; } @Override public String toString() { return builder.toString(); } } }