Integrate Rust frontend into JPlag CLI, add more token types

README.md

@@ -48,7 +48,7 @@ Usage: JPlag [ options ] [ <root-dir> ... ] [ -new <new-dir> ... ] [ -old <old-d
 
 named arguments:
   -h, --help       show this help message and exit
-  -l               {java,python3,cpp,csharp,golang,kotlin,rlang,char,text,scheme} Select the language to parse the submissions (default: java)
+  -l               {java,python3,cpp,csharp,golang,kotlin,rlang,rust,char,text,scheme} Select the language to parse the submissions (default: java)
   -bc BC           Path of the directory containing the base code (common framework used in all submissions)
   -v               {quiet,long} Verbosity of the logging (default: quiet)
   -d               Debug parser. Non-parsable files will be stored (default: false)

jplag.cli/pom.xml

@@ -38,12 +38,20 @@
         </dependency>
         <dependency>
             <groupId>de.jplag</groupId>
-            <artifactId>rlang</artifactId>
+            <artifactId>golang</artifactId>
         </dependency>
         <dependency>
             <groupId>de.jplag</groupId>
             <artifactId>kotlin</artifactId>
         </dependency>
+        <dependency>
+            <groupId>de.jplag</groupId>
+            <artifactId>rlang</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>de.jplag</groupId>
+            <artifactId>rust</artifactId>
+        </dependency>
         <dependency>
             <groupId>de.jplag</groupId>
             <artifactId>scheme</artifactId>

jplag.frontend.rust/src/main/java/de/jplag/rust/JplagRustListener.java

@@ -213,7 +213,7 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
 
     @Override
     public void enterFunction_(RustParser.Function_Context ctx) {
-        Token fn = ((TerminalNodeImpl) ctx.getChild(1)).getSymbol();
+        Token fn = ctx.getChild(TerminalNodeImpl.class, 0).getSymbol();
         transformToken(FUNCTION, fn);
         enterBlockContext(RustBlockContext.FUNCTION_BODY);
         super.enterFunction_(ctx);
@@ -255,6 +255,39 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
         super.exitExpressionWithBlock(ctx);
     }
 
+    @Override
+    public void enterIfExpression(RustParser.IfExpressionContext ctx) {
+        transformToken(IF_STATEMENT, ctx.getStart());
+        enterBlockContext(RustBlockContext.IF_BODY);
+        super.enterIfExpression(ctx);
+    }
+
+    @Override
+    public void exitIfExpression(RustParser.IfExpressionContext ctx) {
+        expectAndLeave(RustBlockContext.IF_BODY);
+        super.exitIfExpression(ctx);
+    }
+
+    @Override
+    public void enterLoopLabel(RustParser.LoopLabelContext ctx) {
+        transformToken(LABEL, ctx.getStart());
+        super.enterLoopLabel(ctx);
+    }
+
+    @Override
+    public void enterInfiniteLoopExpression(RustParser.InfiniteLoopExpressionContext ctx) {
+        Token loopKeyword = ctx.getChild(TerminalNodeImpl.class, 0).getSymbol();
+        transformToken(LOOP_STATEMENT, loopKeyword);
+        enterBlockContext(RustBlockContext.LOOP_BODY);
+        super.enterInfiniteLoopExpression(ctx);
+    }
+
+    @Override
+    public void exitInfiniteLoopExpression(RustParser.InfiniteLoopExpressionContext ctx) {
+        expectAndLeave(RustBlockContext.LOOP_BODY);
+        super.exitInfiniteLoopExpression(ctx);
+    }
+
     @Override
     public void enterCompoundAssignOperator(RustParser.CompoundAssignOperatorContext ctx) {
         transformToken(ASSIGNMENT, ctx.getStart());
@@ -328,6 +361,7 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
         FUNCTION_BODY(FUNCTION_BODY_START, FUNCTION_BODY_END),
         STRUCT_BODY(STRUCT_BODY_BEGIN, STRUCT_BODY_END),
         IF_BODY(IF_BODY_START, IF_BODY_END),
+        LOOP_BODY(LOOP_BODY_START, LOOP_BODY_END),
         INNER_BLOCK(INNER_BLOCK_START, INNER_BLOCK_END),
         USE_TREE(NONE, NONE),
         ATTRIBUTE_TREE(NONE, NONE),
@@ -345,7 +379,7 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
         private final int startType;
         private final int endType;
 
-        <T extends ParserRuleContext> RustBlockContext(int startType, int endType) {
+        RustBlockContext(int startType, int endType) {
             this.startType = startType;
             this.endType = endType;
         }

jplag.frontend.rust/src/main/java/de/jplag/rust/Language.java

@@ -1,10 +1,9 @@
 package de.jplag.rust;
 
-import java.io.File;
-
-import de.jplag.ErrorConsumer;
 import de.jplag.TokenList;
 
+import java.io.File;
+
 public class Language implements de.jplag.Language {
 
     public static final String[] FILE_EXTENSIONS = {".rs"};
@@ -14,8 +13,8 @@ public class Language implements de.jplag.Language {
 
     private final RustParserAdapter parserAdapter;
 
-    public Language(ErrorConsumer consumer) {
-        this.parserAdapter = new RustParserAdapter(consumer);
+    public Language() {
+        this.parserAdapter = new RustParserAdapter();
     }
 
     @Override
@@ -48,23 +47,4 @@ public class Language implements de.jplag.Language {
         return parserAdapter.hasErrors();
     }
 
-    @Override
-    public boolean supportsColumns() {
-        return true;
-    }
-
-    @Override
-    public boolean isPreformatted() {
-        return true;
-    }
-
-    @Override
-    public boolean usesIndex() {
-        return false;
-    }
-
-    @Override
-    public int numberOfTokens() {
-        return RustTokenConstants.NUMBER_DIFF_TOKENS;
-    }
 }

jplag.frontend.rust/src/main/java/de/jplag/rust/RustParserAdapter.java

@@ -1,35 +1,25 @@
 package de.jplag.rust;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-
+import de.jplag.AbstractParser;
+import de.jplag.TokenList;
+import de.jplag.rust.grammar.RustLexer;
+import de.jplag.rust.grammar.RustParser;
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
-import de.jplag.AbstractParser;
-import de.jplag.ErrorConsumer;
-import de.jplag.TokenList;
-import de.jplag.rust.RustTokenConstants.*;
-import de.jplag.rust.grammar.RustLexer;
-import de.jplag.rust.grammar.RustParser;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
 
 public class RustParserAdapter extends AbstractParser {
 
+    private static final int NOT_SET = -1;
     private String currentFile;
     private TokenList tokens;
 
-    /**
-     * Creates the RustParserAdapter
-     * @param consumer the ErrorConsumer that parser errors are passed on to.
-     */
-    public RustParserAdapter(ErrorConsumer consumer) {
-        super(consumer);
-    }
-
     /**
      * Parsers a list of files into a single {@link TokenList}.
      * @param directory the directory of the files.
@@ -43,7 +33,7 @@ public class RustParserAdapter extends AbstractParser {
             if (!parseFile(directory, fileName)) {
                 errors++;
             }
-            tokens.addToken(new RustToken(RustTokenConstants.FILE_END, fileName, -1, -1, -1));
+            tokens.addToken(new RustToken(RustTokenConstants.FILE_END, fileName, NOT_SET, NOT_SET, NOT_SET));
         }
         return tokens;
     }
@@ -69,7 +59,7 @@ public class RustParserAdapter extends AbstractParser {
                 treeWalker.walk(new JplagRustListener(this), parseTree);
             }
         } catch (IOException exception) {
-            getErrorConsumer().addError("Parsing Error in '" + fileName + "':" + File.separator + exception);
+            logger.error("Parsing Error in '" + fileName + "':" + File.separator, exception);
             return false;
         }
         return true;

jplag.frontend.rust/src/main/java/de/jplag/rust/RustToken.java

@@ -17,14 +17,62 @@ public class RustToken extends Token {
             case OUTER_ATTRIBUTE -> "OUTER_ATTR";
             case USE_DECLARATION -> "USE";
             case USE_ITEM -> "USE_ITEM";
-            case STRUCT_BODY_BEGIN -> "STRUCT{";
-            case STRUCT_BODY_END -> "}STRUCT";
+
+            case MODULE -> "MODULE";
+            case MODULE_START -> "MODULE{";
+            case MODULE_END -> "}MODULE";
+
             case FUNCTION -> "FUNCTION";
             case TYPE_PARAMETER -> "<T>";
             case FUNCTION_PARAMETER -> "PARAM";
             case FUNCTION_BODY_START -> "FUNC{";
             case FUNCTION_BODY_END -> "}FUNC";
 
+            case STRUCT -> "STRUCT";
+            case STRUCT_BODY_BEGIN -> "STRUCT{";
+            case STRUCT_BODY_END -> "}STRUCT";
+            case STRUCT_FIELD -> "FIELD";
+
+            case UNION -> "UNION";
+            case UNION_BODY_START -> "UNION{";
+            case UNION_BODY_END -> "}UNION";
+            
+            case TRAIT -> "TRAIT";
+            case TRAIT_BODY_START -> "TRAIT{";
+            case TRAIT_BODY_END -> "}TRAIT";
+
+            case IMPL -> "IMPL";
+            case IMPL_BODY_START -> "IMPL{";
+            case IMPL_BODY_END -> "}IMPL";
+
+            case ENUM -> "ENUM";
+            case ENUM_BODY_START -> "ENUM{";
+            case ENUM_BODY_END -> "}ENUM";
+
+            case MACRO_RULES_DEFINITION -> "MACRO_RULES";
+            case MACRO_RULES_DEFINITION_BODY_START -> "MACRO_RULES{";
+            case MACRO_RULES_DEFINITION_BODY_END -> "}MACRO_RULES";
+
+            case MACRO_RULE -> "MACRO_RULE";
+            case MACRO_RULE_BODY_START -> "MACRO_RULE{";
+            case MACRO_RULE_BODY_END -> "}MACRO_RULE";
+
+            case MACRO_INVOCATION -> "MACRO()";
+            case MACRO_INVOCATION_BODY_START -> "MACRO(){";
+            case MACRO_INVOCATION_BODY_END -> "}MACRO()";
+
+            case EXTERN_BLOCK -> "EXTERN";
+            case EXTERN_BLOCK_START -> "EXTERN{";
+            case EXTERN_BLOCK_END -> "}EXTERN";
+
+            case IF_STATEMENT -> "IF";
+            case IF_BODY_START -> "IF{";
+            case IF_BODY_END -> "}IF";
+
+            case LOOP_STATEMENT -> "LOOP";
+            case LOOP_BODY_START -> "LOOP{";
+            case LOOP_BODY_END -> "}LOOP";
+            
             case INNER_BLOCK_START -> "INNER{";
             case INNER_BLOCK_END -> "}INNER";
 

jplag.frontend.rust/src/main/java/de/jplag/rust/RustTokenConstants.java

@@ -5,8 +5,6 @@ import de.jplag.TokenConstants;
 public interface RustTokenConstants extends TokenConstants {
     int NONE = -1;
 
-    // TOP LEVEL ELEMENTS
-
     int INNER_ATTRIBUTE = 2;
     int OUTER_ATTRIBUTE = 3;
 
@@ -61,16 +59,22 @@ public interface RustTokenConstants extends TokenConstants {
     int EXTERN_BLOCK_START = 41;
     int EXTERN_BLOCK_END = 42;
 
-    int IF_BODY_START = 43;
-    int IF_BODY_END = 44;
+    int IF_STATEMENT = 43;
+    int IF_BODY_START = 44;
+    int IF_BODY_END = 45;
+
+    int LABEL = 46;
+    int LOOP_STATEMENT = 47;
+    int LOOP_BODY_START = 48;
+    int LOOP_BODY_END = 49;
 
-    int INNER_BLOCK_START = 45;
-    int INNER_BLOCK_END = 46;
+    int INNER_BLOCK_START = 50;
+    int INNER_BLOCK_END = 51;
 
-    int ASSIGNMENT = 47;
+    int ASSIGNMENT = 52;
 
-    int VARIABLE_DECLARATION = 48;
+    int VARIABLE_DECLARATION = 53;
 
-    int NUMBER_DIFF_TOKENS = 49;
+    int NUMBER_DIFF_TOKENS = 54;
 
 }

jplag.frontend.rust/src/test/java/de/jplag/rust/RustFrontendTest.java

@@ -1,26 +1,25 @@
 package de.jplag.rust;
 
-import static org.junit.jupiter.api.Assertions.assertArrayEquals;
-import static org.junit.jupiter.api.Assertions.fail;
+import de.jplag.Token;
+import de.jplag.TokenConstants;
+import de.jplag.TokenList;
+import de.jplag.TokenPrinter;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.IntStream;
 import java.util.stream.StreamSupport;
 
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import de.jplag.Token;
-import de.jplag.TokenConstants;
-import de.jplag.TokenList;
-import de.jplag.TokenPrinter;
-import de.jplag.testutils.TestErrorConsumer;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.fail;
 
 public class RustFrontendTest {
 
@@ -37,6 +36,7 @@ public class RustFrontendTest {
     private static final String COMPLETE_TEST_FILE = "Complete.rs";
     public static final int NOT_SET = -1;
     private static final String RUST_SHEBANG = "#!.*$";
+    private static final double EPSILON = 1E-6;
 
     private final Logger logger = LoggerFactory.getLogger("Rust frontend test");
     private final String[] testFiles = new String[] {COMPLETE_TEST_FILE};
@@ -45,8 +45,7 @@ public class RustFrontendTest {
 
     @BeforeEach
     void setup() {
-        TestErrorConsumer consumer = new TestErrorConsumer();
-        language = new Language(consumer);
+        language = new Language();
     }
 
     @Test
@@ -72,27 +71,33 @@ public class RustFrontendTest {
 
         try {
             List<String> lines = Files.readAllLines(testFile.toPath());
-            String emptyLineExpression = SINGLE_LINE_COMMENT();
 
             // All lines that contain code
-            var codeLines = getCodeLines(lines);
+            var codeLines = new ArrayList<>(getCodeLines(lines));
             // All lines that contain token
-            var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().toArray();
+            var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().boxed().toList();
 
-            if (codeLines.length > tokenLines.length) {
-                var diffLine = IntStream.range(0, codeLines.length)
-                        .dropWhile(lineIndex -> lineIndex < tokenLines.length && codeLines[lineIndex] == tokenLines[lineIndex]).findFirst();
-                diffLine.ifPresent(
-                        lineIdx -> fail("Line %d of file '%s' is not represented in the token list.".formatted(codeLines[lineIdx], fileName)));
+            // Keep only lines that have no tokens
+            codeLines.removeAll(tokenLines);
+
+            double coverage = 1.d - (codeLines.size() * 1.d / (codeLines.size() + tokenLines.size()));
+            if (coverage == 1) {
+                logger.info("All lines covered.");
+            } else {
+                logger.info("Coverage: %.1f%%.".formatted(coverage * 100));
+                logger.info("Missing lines {}", codeLines);
+                if (coverage - 0.9 <= EPSILON) {
+                    fail("Source coverage is unsatisfactory");
+                }
             }
-            assertArrayEquals(codeLines, tokenLines);
+
         } catch (IOException exception) {
             logger.info("Error while reading test file %s".formatted(fileName), exception);
             fail();
         }
     }
 
-    private int[] getCodeLines(List<String> lines) {
+    private List<Integer> getCodeLines(List<String> lines) {
         var state = new Object() {
             boolean insideMultilineComment = false;
 
@@ -113,7 +118,7 @@ public class RustFrontendTest {
             } else {
                 return !state.insideMultilineComment;
             }
-        }).toArray();
+        }).boxed().toList();
     }
 
     /**
@@ -135,7 +140,7 @@ public class RustFrontendTest {
         assertArrayEquals(allTokens, foundTokens);
     }
 
-    private static String SINGLE_LINE_COMMENT() {
+    private static String getSingleLineCommentPattern() {
         return RUST_EMPTY_OR_SINGLE_LINE_COMMENT;
     }
 

jplag/src/main/java/de/jplag/options/LanguageOption.java

@@ -16,6 +16,7 @@ public enum LanguageOption {
     GO_LANG("golang", "de.jplag.golang.Language"),
     KOTLIN("kotlin", "de.jplag.kotlin.Language"),
     R_LANG("rlang", "de.jplag.rlang.Language"),
+    RUST("rust", "de.jplag.rust.Language"),
     CHAR("char", "de.jplag.chars.Language"),
     TEXT("text", "de.jplag.text.Language"),
     SCHEME("scheme", "de.jplag.scheme.Language");

pom.xml

@@ -172,6 +172,11 @@
                 <artifactId>rlang</artifactId>
                 <version>${revision}</version>
             </dependency>
+            <dependency>
+                <groupId>de.jplag</groupId>
+                <artifactId>rust</artifactId>
+                <version>${revision}</version>
+            </dependency>
             <dependency>
                 <groupId>de.jplag</groupId>
                 <artifactId>scheme</artifactId>