Fix Sonarcloud code smells, fix tuples, handle "if let"

jplag.frontend.rust/README.md

@@ -37,9 +37,7 @@ or _etcetera_ pattern `..` is used to skip a number of elements, so that the ele
 the assigned object.
 
 These `let` pattern assignments can be replaced with a sequence of more basic assignments. This is a possible
-vulnerability of this frontend. 
-
-[...]
+vulnerability of this frontend.
 
 #### Problem in Rust (2): `return` is optional
 
@@ -68,7 +66,7 @@ On the other hand, "the last expression of a block evaluated" does not hold the
 return
 statement.
 
-For the moment, implicit block values are neglected.
+For the moment, implicit block values get no special tokens.
 
 #### Problem in Rust (3): Macros
 
@@ -76,7 +74,8 @@ Macros are a vital part of Rust. They allow to expand brief statements into more
 
 The expansion of the macro arguments into the macro code and the expansion of the macro code itself are purely textual, so a Rust parser does not parse their syntax (apart from the bracket structure). This makes it hard to generate meaningful tokens for them.
 
-[...]
+Currently, macro rule definition bodies and macro macro invocation arguments/bodies get no tokens.
+
 ### Usage
 
 To use the Rust frontend, add the `-l rust` flag in the CLI, or use a `JPlagOption` object set

jplag.frontend.rust/src/main/java/de/jplag/rust/JplagRustListener.java

@@ -10,6 +10,8 @@ import org.antlr.v4.runtime.tree.*;
 import de.jplag.rust.grammar.RustParser;
 import de.jplag.rust.grammar.RustParserBaseListener;
 
+import java.util.Objects;
+
 public class JplagRustListener extends RustParserBaseListener implements ParseTreeListener {
 
     private final RustParserAdapter parserAdapter;
@@ -137,16 +139,22 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
     }
 
     @Override
-    public void enterTupleElements(RustParser.TupleElementsContext context) {
-        if (context.getChildCount() <= 2)
+    public void enterTupleExpression(RustParser.TupleExpressionContext context) {
+        state.enter(RustContext.TUPLE);
+
+        var elements = context.getChild(RustParser.TupleElementsContext.class, 0);
+        // one child = exactly one subtree and no trailing comma
+        if (Objects.nonNull(elements) && 0 < elements.getChildCount() && elements.getChildCount() == 1)
             state.enter(RustContext.REDUNDANT_TUPLE);
-        super.enterTupleElements(context);
+
+        super.enterTupleExpression(context);
     }
 
     @Override
-    public void exitTupleElements(RustParser.TupleElementsContext context) {
+    public void exitTupleExpression(RustParser.TupleExpressionContext ctx) {
         state.maybeLeave(RustContext.REDUNDANT_TUPLE);
-        super.exitTupleElements(context);
+        state.leave(RustContext.TUPLE);
+        super.exitTupleExpression(ctx);
     }
 
     @Override
@@ -255,6 +263,18 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
         super.exitEnumeration(context);
     }
 
+    @Override
+    public void enterEnumItemTuple(RustParser.EnumItemTupleContext ctx) {
+        state.enter(RustContext.TUPLE);
+        super.enterEnumItemTuple(ctx);
+    }
+
+    @Override
+    public void exitEnumItemTuple(RustParser.EnumItemTupleContext ctx) {
+        state.leave(RustContext.TUPLE);
+        super.exitEnumItemTuple(ctx);
+    }
+
     @Override
     public void enterEnumItem(RustParser.EnumItemContext context) {
         transformToken(ENUM_ITEM, context.getStart());
@@ -397,6 +417,20 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
         super.exitIfExpression(context);
     }
 
+    @Override
+    public void enterIfLetExpression(RustParser.IfLetExpressionContext ctx) {
+        transformToken(IF_STATEMENT, ctx.getStart());
+        state.enter(RustContext.IF_BODY);
+        super.enterIfLetExpression(ctx);
+    }
+
+    @Override
+    public void exitIfLetExpression(RustParser.IfLetExpressionContext ctx) {
+        state.maybeLeave(RustContext.ELSE_BODY);
+        state.leave(RustContext.IF_BODY);
+        super.exitIfLetExpression(ctx);
+    }
+
     @Override
     public void enterLoopLabel(RustParser.LoopLabelContext context) {
         transformToken(LABEL, context.getStart());
@@ -490,12 +524,6 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
         super.enterMatchArmGuard(context);
     }
 
-    @Override
-    public void enterRangeExpression(RustParser.RangeExpressionContext context) {
-        // Ranges are ignored for now.
-        super.enterRangeExpression(context);
-    }
-
     @Override
     public void enterCompoundAssignOperator(RustParser.CompoundAssignOperatorContext context) {
         transformToken(ASSIGNMENT, context.getStart());
@@ -515,9 +543,9 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
     }
 
     @Override
-    public void enterMethodCallExpression(RustParser.MethodCallExpressionContext ctx) {
+    public void enterMethodCallExpression(RustParser.MethodCallExpressionContext context) {
         state.enter(RustContext.CALL);
-        super.enterMethodCallExpression(ctx);
+        super.enterMethodCallExpression(context);
     }
 
     @Override
@@ -600,6 +628,8 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
         switch (state.getCurrent()) {
             case TUPLE_STRUCT_PATTERN -> transformToken(STRUCT_FIELD, context.getStart());
             case TUPLE_PATTERN -> transformToken(TUPLE_ELEMENT, context.getStart());
+            default -> {
+            }
         }
         super.enterPattern(context);
     }
@@ -607,47 +637,59 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
     @Override
     public void visitTerminal(TerminalNode node) {
         final Token token = node.getSymbol();
+        final ParseTree parentNode = node.getParent();
+        RustContext stateContext = state.getCurrent();
         switch (node.getText()) {
             case "*" -> {
-                if (node.getParent() instanceof RustParser.UseTreeContext) {
+                if (parentNode instanceof RustParser.UseTreeContext) {
                     transformToken(USE_ITEM, token);
                 }
             }
-            case "let" -> transformToken(VARIABLE_DECLARATION, token);
+            case "let" -> {
+                if (stateContext != RustContext.MACRO_INNER) {
+                    transformToken(VARIABLE_DECLARATION, token);
+                }
+            }
             case "=" -> {
-                if (!(node.getParent() instanceof RustParser.AttrInputContext || node.getParent() instanceof RustParser.TypeParamContext
-                        || node.getParent() instanceof RustParser.GenericArgsBindingContext)) {
+                if (!(parentNode instanceof RustParser.AttrInputContext || parentNode instanceof RustParser.TypeParamContext
+                        || parentNode instanceof RustParser.GenericArgsBindingContext) && stateContext != RustContext.MACRO_INNER) {
+                    transformToken(ASSIGNMENT, token);
+                }
+            }
+            case ":" -> {
+                if (parentNode instanceof RustParser.StructExprFieldContext) {
                     transformToken(ASSIGNMENT, token);
                 }
             }
             case "{" -> {
-                int startType = state.getCurrent().getStartType();
+                int startType = stateContext.getStartType();
                 if (startType != NONE) {
                     transformToken(startType, token);
                 }
-                switch (state.getCurrent()) {
-                    case MACRO_RULES_DEFINITION_BODY, MACRO_INVOCATION_BODY, MACRO_INNER -> state.enter(RustContext.MACRO_INNER);
+                switch (stateContext) {
+                    case MACRO_RULE_BODY, MACRO_INVOCATION_BODY, MACRO_INNER -> state.enter(RustContext.MACRO_INNER);
+                    default -> {
+                    }
                 }
 
             }
             case "}" -> {
-                int endType = state.getCurrent().getEndType();
+                int endType = stateContext.getEndType();
                 if (endType != NONE) {
                     transformToken(endType, token);
                 }
 
-                if (state.getCurrent() == RustContext.MACRO_INNER) {
+                if (stateContext == RustContext.MACRO_INNER) {
                     // maybe this is the end of a macro invocation/definition
                     state.leave(RustContext.MACRO_INNER);
-                    if (state.getCurrent() == RustContext.MACRO_INVOCATION_BODY) {
-                        transformToken(MACRO_INVOCATION_BODY_END, token);
-                    } else if (state.getCurrent() == RustContext.MACRO_RULES_DEFINITION_BODY) {
-                        transformToken(MACRO_RULES_DEFINITION_BODY_END, token);
+                    stateContext = state.getCurrent();
+                    if (stateContext == RustContext.MACRO_INVOCATION_BODY || stateContext == RustContext.MACRO_RULE_BODY) {
+                        transformToken(stateContext.getEndType(), token);
                     }
                 }
             }
             case "(" -> {
-                switch (state.getCurrent()) {
+                switch (stateContext) {
                     case STRUCT_BODY -> transformToken(RustContext.STRUCT_BODY.getStartType(), token);
                     case TUPLE -> transformToken(RustContext.TUPLE.getStartType(), token);
                     case MACRO_INVOCATION_BODY -> {
@@ -656,25 +698,58 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
                     }
                     case MACRO_INNER -> state.enter(RustContext.MACRO_INNER);
                     case CALL -> transformToken(APPLY, token);
+                    default -> {
+                    }
                 }
             }
             case ")" -> {
-                switch (state.getCurrent()) {
+                switch (stateContext) {
                     case STRUCT_BODY -> transformToken(RustContext.STRUCT_BODY.getEndType(), token);
                     case TUPLE -> transformToken(RustContext.TUPLE.getEndType(), token);
                     case MACRO_INVOCATION_BODY -> {
-                        /* do nothing */ }
+                        /* do nothing */
+                    }
                     case MACRO_INNER -> {
                         state.leave(RustContext.MACRO_INNER);
-                        if (state.getCurrent() == RustContext.MACRO_INVOCATION_BODY) {
+                        stateContext = state.getCurrent();
+                        if (stateContext == RustContext.MACRO_INVOCATION_BODY) {
                             transformToken(MACRO_INVOCATION_BODY_END, token);
                         }
                     }
+                    default -> {
+                    }
 
                 }
             }
+            case "[" -> {
+                switch (stateContext) {
+                    case MACRO_INVOCATION_BODY -> {
+                        transformToken(MACRO_INVOCATION_BODY_START, token);
+                        state.enter(RustContext.MACRO_INNER);
+                    }
+                    case MACRO_INNER -> state.enter(RustContext.MACRO_INNER);
+                    default -> {
+                    }
+                }
+            }
+            case "]" -> {
+                switch (stateContext) {
+                    case MACRO_INVOCATION_BODY -> {
+                        /* do nothing */
+                    }
+                    case MACRO_INNER -> {
+                        state.leave(RustContext.MACRO_INNER);
+                        stateContext = state.getCurrent();
+                        if (stateContext == RustContext.MACRO_INVOCATION_BODY) {
+                            transformToken(MACRO_INVOCATION_BODY_END, token);
+                        }
+                    }
+                    default -> {
+                    }
+                }
+            }
             case "else" -> {
-                if (state.getCurrent() == RustContext.IF_BODY) {
+                if (stateContext == RustContext.IF_BODY) {
                     transformToken(ELSE_STATEMENT, token);
                     state.enter(RustContext.ELSE_BODY);
                 }
@@ -703,17 +778,16 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
 
     @Override
     public void enterGenericArg(RustParser.GenericArgContext context) {
-        transformToken(TYPE_ARGUMENT, context.getStart());
+        // Only type arguments for methods, not for type expressions
+        if (context.getParent().getParent() instanceof RustParser.PathInExpressionContext) {
+            transformToken(TYPE_ARGUMENT, context.getStart());
+        }
         super.enterGenericArg(context);
     }
 
     @Override
-    public void visitErrorNode(ErrorNode node) {
-
-    }
-
-    @Override
-    public void enterEveryRule(ParserRuleContext context) {
+    public void
+    enterEveryRule(ParserRuleContext context) {
         // ExpressionContext gets no own enter/exit method
         // used in various 'lists' of elements
         if (context instanceof RustParser.ExpressionContext expression) {
@@ -734,10 +808,9 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
 
     @Override
     public void exitEveryRule(ParserRuleContext context) {
-        if (context instanceof RustParser.ExpressionContext) {
-            if (context.parent instanceof RustParser.ClosureExpressionContext) {
-                transformToken(CLOSURE_BODY_END, context.getStop());
-            }
+        if (context instanceof RustParser.ExpressionContext && context.parent instanceof RustParser.ClosureExpressionContext) {
+            transformToken(CLOSURE_BODY_END, context.getStop());
+
         }
     }
 
@@ -745,7 +818,9 @@ public class JplagRustListener extends RustParserBaseListener implements ParseTr
      * Implementation of Context for the Rust language
      */
     enum RustContext implements ParserState.Context {
-        /** This is used to make sure that the stack is not empty -> getCurrent() != null **/
+        /**
+         * This is used to make sure that the stack is not empty -> getCurrent() != null
+         **/
         FILE(NONE, NONE),
 
         /**

jplag.frontend.rust/src/main/java/de/jplag/rust/Language.java

@@ -6,7 +6,7 @@ import de.jplag.TokenList;
 
 public class Language implements de.jplag.Language {
 
-    public static final String[] FILE_EXTENSIONS = {".rs"};
+    protected static final String[] FILE_EXTENSIONS = {".rs"};
     public static final String NAME = "Rust frontend";
     public static final String SHORT_NAME = "Rust";
     public static final int MINIMUM_TOKEN_MATCH = 8;

jplag.frontend.rust/src/main/java/de/jplag/rust/ParserState.java

@@ -31,7 +31,7 @@ public class ParserState<C extends ParserState.Context> {
      * @param contexts The contexts to expect to end here
      */
     @SafeVarargs
-    final protected void leave(C... contexts) {
+    protected final void leave(C... contexts) {
         C topContext = blockContexts.pop();
         assert Arrays.stream(contexts).anyMatch(context -> context == topContext);
     }

jplag.frontend.rust/src/main/java/de/jplag/rust/RustParserAdapter.java

@@ -4,6 +4,7 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 
+import de.jplag.TokenConstants;
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.antlr.v4.runtime.ParserRuleContext;
@@ -34,7 +35,7 @@ public class RustParserAdapter extends AbstractParser {
             if (!parseFile(directory, fileName)) {
                 errors++;
             }
-            tokens.addToken(new RustToken(RustTokenConstants.FILE_END, fileName, NOT_SET, NOT_SET, NOT_SET));
+            tokens.addToken(new RustToken(TokenConstants.FILE_END, fileName, NOT_SET, NOT_SET, NOT_SET));
         }
         return tokens;
     }
@@ -46,9 +47,9 @@ public class RustParserAdapter extends AbstractParser {
 
             // create a lexer, a parser and a buffer between them.
             RustLexer lexer = new RustLexer(CharStreams.fromStream(inputStream));
-            CommonTokenStream tokens = new CommonTokenStream(lexer);
+            CommonTokenStream tokenStream = new CommonTokenStream(lexer);
 
-            RustParser parser = new RustParser(tokens);
+            RustParser parser = new RustParser(tokenStream);
 
             // Create a tree walker and the entry context defined by the parser grammar
             ParserRuleContext entryContext = parser.crate();

jplag.frontend.rust/src/test/java/de/jplag/rust/RustFrontendTest.java

@@ -1,6 +1,5 @@
 package de.jplag.rust;
 
-import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.fail;
 
 import java.io.File;
@@ -22,7 +21,7 @@ import de.jplag.TokenConstants;
 import de.jplag.TokenList;
 import de.jplag.TokenPrinter;
 
-public class RustFrontendTest {
+class RustFrontendTest {
 
     /**
      * Regular expression for empty lines and single line comments.
@@ -36,6 +35,7 @@ public class RustFrontendTest {
      */
     private static final String COMPLETE_TEST_FILE = "complete.rs";
     public static final int NOT_SET = -1;
+    private static final String EMPTY_STRING = "";
     private static final String RUST_SHEBANG = "#!.*$";
     private static final double EPSILON = 1E-6;
 
@@ -129,17 +129,20 @@ public class RustFrontendTest {
      * @param fileName The file name of the complete code example
      */
     private void testTokenCoverage(TokenList tokens, String fileName) {
-        var foundTokens = StreamSupport.stream(tokens.allTokens().spliterator(), true).mapToInt(Token::getType).sorted().distinct().toArray();
+        var foundTokens = StreamSupport.stream(tokens.allTokens().spliterator(), true).mapToInt(Token::getType).distinct().boxed().toList();
+        var allTokens = IntStream.range(0, RustTokenConstants.NUMBER_DIFF_TOKENS).boxed().toList();
+        allTokens = new ArrayList<>(allTokens);
+
+        // Only non-found tokens are left
+        allTokens.removeAll(foundTokens);
         // Exclude SEPARATOR_TOKEN, as it does not occur
-        var allTokens = IntStream.range(0, RustTokenConstants.NUMBER_DIFF_TOKENS).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).toArray();
+        allTokens.remove((Integer) (TokenConstants.SEPARATOR_TOKEN));
 
-        if (allTokens.length > foundTokens.length) {
-            var diffLine = IntStream.range(0, allTokens.length)
-                    .dropWhile(lineIndex -> lineIndex < foundTokens.length && allTokens[lineIndex] == foundTokens[lineIndex]).findFirst();
-            diffLine.ifPresent(lineIdx -> fail("Token type %s was not found in the complete code example '%s'."
-                    .formatted(new RustToken(allTokens[lineIdx], fileName, NOT_SET, NOT_SET, NOT_SET).type2string(), fileName)));
+        if (!allTokens.isEmpty()) {
+            var notFoundTypes = allTokens.stream().map(type -> new RustToken(type, EMPTY_STRING, NOT_SET, NOT_SET, NOT_SET).type2string()).toList();
+            fail("Some %d token types were not found in the complete code example '%s':\n%s".formatted(notFoundTypes.size(), fileName,
+                    notFoundTypes));
         }
-        assertArrayEquals(allTokens, foundTokens);
     }
 
 }