Add interface tokens, clean up test

jplag.frontend.golang/src/main/java/de/jplag/golang/GoListener.java

@@ -66,7 +66,7 @@ public class GoListener extends GoParserBaseListener {
 
     @Override
     public void enterImportDecl(GoParser.ImportDeclContext context) {
-        transformToken(IMPORT_DECL, context.getStart());
+        transformToken(IMPORT_DECLARATION, context.getStart());
 
         // if the children contain TerminalNodes, then it must be '(' and ')'
         Optional<TerminalNode> listStart = context.children.stream().filter(tree -> tree instanceof TerminalNode).map(TerminalNode.class::cast)
@@ -90,11 +90,26 @@ public class GoListener extends GoParserBaseListener {
         super.enterImportSpec(context);
     }
 
+    /* INTERFACE */
+
+    @Override
+    public void enterInterfaceType(GoParser.InterfaceTypeContext context) {
+        transformToken(INTERFACE_DECLARATION, context.getStart());
+        enterContext(GoBlockContext.INTERFACE_BODY);
+        super.enterInterfaceType(context);
+    }
+
+    @Override
+    public void exitInterfaceType(GoParser.InterfaceTypeContext context) {
+        expectAndLeave(GoBlockContext.INTERFACE_BODY);
+        super.exitInterfaceType(context);
+    }
+
     /* STRUCT */
 
     @Override
     public void enterStructType(GoParser.StructTypeContext context) {
-        transformToken(STRUCT_DECLARATION_BEGIN, context.getStart());
+        transformToken(STRUCT_DECLARATION, context.getStart());
         enterContext(GoBlockContext.STRUCT_BODY);
         super.enterStructType(context);
     }
@@ -141,7 +156,11 @@ public class GoListener extends GoParserBaseListener {
 
     @Override
     public void enterParameterDecl(GoParser.ParameterDeclContext context) {
-        transformToken(FUNCTION_PARAMETER, context.getStart(), context.getStop());
+        if (context.parent.parent instanceof GoParser.ReceiverContext) {
+            transformToken(RECEIVER, context.getStart(), context.getStop());
+        } else {
+            transformToken(FUNCTION_PARAMETER, context.getStart(), context.getStop());
+        }
         super.enterParameterDecl(context);
     }
 
@@ -374,10 +393,24 @@ public class GoListener extends GoParserBaseListener {
         if (context.parent.parent instanceof GoParser.CompositeLitContext) {
             transformToken(NAMED_TYPE_CONSTRUCTOR, context.getStart());
             enterContext(GoBlockContext.NAMED_TYPE_BODY);
+        } else if (context.parent instanceof GoParser.InterfaceTypeContext) {
+            transformToken(TYPE_CONSTRAINT, context.getStart(), context.getStop());
         }
         super.enterTypeName(context);
     }
 
+    @Override
+    public void enterTypeAssertion(GoParser.TypeAssertionContext context) {
+        transformToken(TYPE_ASSERTION, context.getStart(), context.getStop());
+        super.enterTypeAssertion(context);
+    }
+
+    @Override
+    public void enterMethodSpec(GoParser.MethodSpecContext context) {
+        transformToken(INTERFACE_METHOD, context.getStart(), context.getStop());
+        super.enterMethodSpec(context);
+    }
+
     /* CONTROL FLOW KEYWORDS */
 
     @Override
@@ -462,7 +495,8 @@ public class GoListener extends GoParserBaseListener {
         SWITCH_BLOCK(SWITCH_BLOCK_BEGIN, SWITCH_BLOCK_END, NONE),
         SELECT_CONTEXT(SELECT_BLOCK_BEGIN, SELECT_BLOCK_END, NONE),
         STATEMENT_BLOCK(STATEMENT_BLOCK_BEGIN, STATEMENT_BLOCK_END, NONE),
-        CASE_BLOCK(CASE_BLOCK_BEGIN, CASE_BLOCK_END, NONE);
+        CASE_BLOCK(CASE_BLOCK_BEGIN, CASE_BLOCK_END, NONE),
+        INTERFACE_BODY(INTERFACE_BLOCK_BEGIN, INTERFACE_BLOCK_END, NONE);
 
         private final int beginTokenType;
         private final int endTokenType;

jplag.frontend.golang/src/main/java/de/jplag/golang/GoToken.java

@@ -20,28 +20,39 @@ public class GoToken extends Token {
             case IMPORT_CLAUSE -> "IMPORT";
             case IMPORT_CLAUSE_BEGIN -> "IMPORT(";
             case IMPORT_CLAUSE_END -> ")IMPORT";
-            case IMPORT_DECL -> "IMPORT_DECL";
+            case IMPORT_DECLARATION -> "IMPORT_DECL";
 
             case ARRAY_BODY_BEGIN -> "ARRAY{";
             case ARRAY_BODY_END -> "}ARRAY";
 
-            case STRUCT_DECLARATION_BEGIN -> "STRUCT";
+            case STRUCT_DECLARATION -> "STRUCT";
             case STRUCT_BODY_BEGIN -> "STRUCT{";
             case STRUCT_BODY_END -> "}STRUCT";
 
+            case INTERFACE_DECLARATION -> "INTERFACE";
+            case INTERFACE_BLOCK_BEGIN -> "INTERFACE{";
+            case INTERFACE_BLOCK_END -> "}INTERFACE";
+
+            case INTERFACE_METHOD -> "INTERFACE_METHOD";
+            case TYPE_CONSTRAINT -> "TYPE_CONSTRAINT";
+            case TYPE_ASSERTION -> "TYPE_ASSERTION";
+
             case MAP_BODY_BEGIN -> "MAP{";
             case MAP_BODY_END -> "}MAP";
 
             case SLICE_BODY_BEGIN -> "SLICE{";
             case SLICE_BODY_END -> "}SLICE";
 
-            case NAMED_TYPE_BODY_BEGIN -> "NAMED_TYPE{";
-            case NAMED_TYPE_BODY_END -> "}NAMED_TYPE";
+            case NAMED_TYPE_BODY_BEGIN -> "CTYPE{";
+            case NAMED_TYPE_BODY_END -> "}CTYPE";
 
             case MEMBER_DECLARATION -> "FIELD";
 
+            // Functions and Methods
+
             case FUNCTION_DECLARATION -> "FUNC";
             case METHOD_DECLARATION -> "METHOD";
+            case RECEIVER -> "RECEIVER";
             case FUNCTION_PARAMETER -> "PARAM";
             case FUNCTION_BODY_BEGIN -> "FUNC{";
             case FUNCTION_BODY_END -> "}FUNC";

jplag.frontend.golang/src/main/java/de/jplag/golang/GoTokenConstants.java

@@ -4,85 +4,95 @@ import de.jplag.TokenConstants;
 
 public interface GoTokenConstants extends TokenConstants {
 
-    // Top level structures
+    // TOP LEVEL STRUCTURES
 
     int PACKAGE = 2;
     int IMPORT_CLAUSE = 3;
     int IMPORT_CLAUSE_BEGIN = 4;
     int IMPORT_CLAUSE_END = 5;
-    int IMPORT_DECL = 6;
+    int IMPORT_DECLARATION = 6;
     int ARRAY_BODY_BEGIN = 7;
     int ARRAY_BODY_END = 8;
-    int STRUCT_DECLARATION_BEGIN = 9;
+    int STRUCT_DECLARATION = 9;
     int STRUCT_BODY_BEGIN = 10;
     int STRUCT_BODY_END = 11;
-    int MAP_BODY_BEGIN = 12;
-    int MAP_BODY_END = 13;
-    int SLICE_BODY_BEGIN = 14;
-    int SLICE_BODY_END = 15;
-    int NAMED_TYPE_BODY_BEGIN = 16;
-    int NAMED_TYPE_BODY_END = 17;
-    int MEMBER_DECLARATION = 18;
-    int FUNCTION_DECLARATION = 19;
-    int METHOD_DECLARATION = 20;
-    int FUNCTION_PARAMETER = 21;
-    int FUNCTION_BODY_BEGIN = 22;
-    int FUNCTION_BODY_END = 23;
-
-    // Control flow statements
-
-    int IF_STATEMENT = 24;
-    int IF_BLOCK_BEGIN = 25;
-    int IF_BLOCK_END = 26;
-    int ELSE_BLOCK_BEGIN = 27;
-    int ELSE_BLOCK_END = 28;
-    int FOR_STATEMENT = 29;
-    int FOR_BLOCK_BEGIN = 30;
-    int FOR_BLOCK_END = 31;
-    int SWITCH_STATEMENT = 32;
-    int SWITCH_BLOCK_BEGIN = 33;
-    int SWITCH_BLOCK_END = 34;
-    int SWITCH_CASE = 35;
-    int SELECT_STATEMENT = 36;
-    int SELECT_BLOCK_BEGIN = 37;
-    int SELECT_BLOCK_END = 38;
-    int CASE_BLOCK_BEGIN = 39;
-    int CASE_BLOCK_END = 40;
+    int INTERFACE_DECLARATION = 12;
+    int INTERFACE_BLOCK_BEGIN = 13;
+    int INTERFACE_BLOCK_END = 14;
+    int INTERFACE_METHOD = 15;
+    int TYPE_CONSTRAINT = 16;
+    int TYPE_ASSERTION = 17;
+    int MAP_BODY_BEGIN = 18;
+    int MAP_BODY_END = 19;
+    int SLICE_BODY_BEGIN = 20;
+    int SLICE_BODY_END = 21;
+    int NAMED_TYPE_BODY_BEGIN = 22;
+    int NAMED_TYPE_BODY_END = 23;
+    int MEMBER_DECLARATION = 24;
+
+    // FUNCTIONS AND METHODS
+
+    int FUNCTION_DECLARATION = 25;
+    int METHOD_DECLARATION = 26;
+    int RECEIVER = 27;
+    int FUNCTION_PARAMETER = 28;
+    int FUNCTION_BODY_BEGIN = 29;
+    int FUNCTION_BODY_END = 30;
+
+    // CONTROL FLOW STATEMENTS
+
+    int IF_STATEMENT = 31;
+    int IF_BLOCK_BEGIN = 32;
+    int IF_BLOCK_END = 33;
+    int ELSE_BLOCK_BEGIN = 34;
+    int ELSE_BLOCK_END = 35;
+    int FOR_STATEMENT = 36;
+    int FOR_BLOCK_BEGIN = 37;
+    int FOR_BLOCK_END = 38;
+    int SWITCH_STATEMENT = 39;
+    int SWITCH_BLOCK_BEGIN = 40;
+    int SWITCH_BLOCK_END = 41;
+    int SWITCH_CASE = 42;
+    int SELECT_STATEMENT = 43;
+    int SELECT_BLOCK_BEGIN = 44;
+    int SELECT_BLOCK_END = 45;
+    int CASE_BLOCK_BEGIN = 46;
+    int CASE_BLOCK_END = 47;
 
     // Statements
 
-    int VARIABLE_DECLARATION = 41;
-    int CONSTANT_DECLARATION = 42;
-    int FUNCTION_LITERAL = 43;
-    int ASSIGNMENT = 44;
-    int SEND_STATEMENT = 45;
-    int RECEIVE_STATEMENT = 46;
-    int INVOCATION = 47;
-    int ARGUMENT = 48;
-    int STATEMENT_BLOCK_BEGIN = 49;
-    int STATEMENT_BLOCK_END = 50;
-
-    // Object creation
-
-    int ARRAY_ELEMENT = 51;
-    int MAP_ELEMENT = 52;
-    int SLICE_ELEMENT = 53;
-    int NAMED_TYPE_ELEMENT = 54;
-    int ARRAY_CONSTRUCTOR = 55;
-    int MAP_CONSTRUCTOR = 56;
-    int SLICE_CONSTRUCTOR = 57;
-    int NAMED_TYPE_CONSTRUCTOR = 58;
-
-    // Control flow keywords
-
-    int RETURN = 59;
-    int BREAK = 60;
-    int CONTINUE = 61;
-    int FALLTHROUGH = 62;
-    int GOTO = 63;
-    int GO = 64;
-    int DEFER = 65;
-
-    int NUM_DIFF_TOKENS = 66;
+    int VARIABLE_DECLARATION = 48;
+    int CONSTANT_DECLARATION = 49;
+    int FUNCTION_LITERAL = 50;
+    int ASSIGNMENT = 51;
+    int SEND_STATEMENT = 52;
+    int RECEIVE_STATEMENT = 53;
+    int INVOCATION = 54;
+    int ARGUMENT = 55;
+    int STATEMENT_BLOCK_BEGIN = 56;
+    int STATEMENT_BLOCK_END = 57;
+
+    // OBJECT CREATION
+
+    int ARRAY_ELEMENT = 58;
+    int MAP_ELEMENT = 59;
+    int SLICE_ELEMENT = 60;
+    int NAMED_TYPE_ELEMENT = 61;
+    int ARRAY_CONSTRUCTOR = 62;
+    int MAP_CONSTRUCTOR = 63;
+    int SLICE_CONSTRUCTOR = 64;
+    int NAMED_TYPE_CONSTRUCTOR = 65;
+
+    // CONTROL FLOW KEYWORDS
+
+    int RETURN = 66;
+    int BREAK = 67;
+    int CONTINUE = 68;
+    int FALLTHROUGH = 69;
+    int GOTO = 70;
+    int GO = 71;
+    int DEFER = 72;
+
+    int NUM_DIFF_TOKENS = 73;
 
 }

jplag.frontend.golang/src/test/java/de/jplag/golang/GoFrontendTest.java

@@ -9,6 +9,7 @@ import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.OptionalInt;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.StreamSupport;
@@ -45,7 +46,7 @@ public class GoFrontendTest {
      */
     private static final String DELIMITED_COMMENT_END = ".*\\*/\\s*$";
     private static final String NOT_SET_STRING = "";
-    private static final int NOT_SET = -1;
+    public static final int NOT_SET = -1;
 
     private final Logger logger = LoggerFactory.getLogger("GoLang frontend test");
     private final String[] testFiles = new String[] {COMPLETE_TEST_FILE};
@@ -96,31 +97,31 @@ public class GoFrontendTest {
     private void testSourceCoverage(String fileName, TokenList tokens) {
         File testFile = new File(testFileLocation, fileName);
 
+        List<String> lines = null;
         try {
-            List<String> lines = Files.readAllLines(testFile.toPath());
-
-            // All lines that contain code
-            var codeLines = getCodeLines(lines);
-            // All lines that contain token
-            var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().boxed().toList();
-
-            if (codeLines.size() > tokenLines.size()) {
-                List<Integer> missedLinesIndices = new ArrayList<>(codeLines);
-                missedLinesIndices.removeAll(tokenLines);
-                var missedLines = missedLinesIndices.stream().filter(index -> !lines.get(index - 1).matches(".*(\\+\\+|--).*")).map(Object::toString)
-                        .collect(Collectors.joining(", "));
-                if (!missedLines.isBlank()) {
-                    fail("Found lines in file '%s' that are not represented in the token list. \n\tMissed lines: %s".formatted(fileName,
-                            missedLines));
-                }
-            }
-            IntStream.range(0, codeLines.size()).dropWhile(index -> Objects.equals(codeLines.get(index), tokenLines.get(index))).findAny()
-                    .ifPresent(i -> fail(
-                            "Not all lines of code in '%s' are represented in tokens, starting with line %d.".formatted(fileName, codeLines.get(i))));
+            lines = Files.readAllLines(testFile.toPath());
         } catch (IOException exception) {
             logger.info("Error while reading test file %s".formatted(fileName), exception);
             fail();
         }
+
+        // All lines that contain code
+        var codeLines = getCodeLines(lines);
+        // All lines that contain a token
+        var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().boxed().toList();
+
+        if (codeLines.size() > tokenLines.size()) {
+            List<Integer> missedLinesIndices = new ArrayList<>(codeLines);
+            missedLinesIndices.removeAll(tokenLines);
+            var missedLines = missedLinesIndices.stream().map(Object::toString).collect(Collectors.joining(", "));
+            if (!missedLines.isBlank()) {
+                fail("Found lines in file '%s' that are not represented in the token list. \n\tMissed lines: %s".formatted(fileName, missedLines));
+            }
+        }
+        OptionalInt differingLine = IntStream.range(0, codeLines.size())
+                .dropWhile(index -> Objects.equals(codeLines.get(index), tokenLines.get(index))).findAny();
+        differingLine.ifPresent(
+                i -> fail("Not all lines of code in '%s' are represented in tokens, starting with line %d.".formatted(fileName, codeLines.get(i))));
     }
 
     /**
@@ -161,16 +162,21 @@ public class GoFrontendTest {
      * @param fileName The file name of the complete code example
      */
     private void testTokenCoverage(TokenList tokens, String fileName) {
-        var foundTokens = StreamSupport.stream(tokens.allTokens().spliterator(), true).mapToInt(Token::getType).sorted().distinct().toArray();
+        var foundTokens = StreamSupport.stream(tokens.allTokens().spliterator(), true).mapToInt(Token::getType).sorted().distinct().boxed().toList();
+
         // Exclude SEPARATOR_TOKEN, as it does not occur
-        var allTokens = IntStream.range(0, language.numberOfTokens()).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).toArray();
+        var missingTokenTypes = IntStream.range(0, language.numberOfTokens()).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).boxed()
+                .collect(Collectors.toList());
+        missingTokenTypes.removeAll(foundTokens);
 
-        if (allTokens.length > foundTokens.length) {
-            var diffLine = IntStream.range(0, allTokens.length)
-                    .dropWhile(lineIdx -> lineIdx < foundTokens.length && allTokens[lineIdx] == foundTokens[lineIdx]).findFirst();
-            diffLine.ifPresent(lineIdx -> fail("Token type %s was not found in the complete code example '%s'."
-                    .formatted(new GoToken(allTokens[lineIdx], fileName, -1, -1, -1).type2string(), fileName)));
+        if (!missingTokenTypes.isEmpty()) {
+            String missingTypesList = missingTokenTypes.stream().map(type -> getDummyToken(type).type2string()).collect(Collectors.joining(", "));
+            fail("Some Token types were not found in the complete code example '%s': %s.".formatted(fileName, missingTypesList));
         }
-        assertArrayEquals(allTokens, foundTokens);
+
+    }
+
+    private GoToken getDummyToken(int type) {
+        return new GoToken(type, NOT_SET_STRING, NOT_SET, NOT_SET, NOT_SET);
     }
 }

jplag.frontend.golang/src/test/resources/de/jplag/golang/Complete.go

@@ -140,4 +140,28 @@ func channelExample(ch chan<- int, value int) {
 func goExample() {
 	ch := make(chan<- int)
 	go innerBlock(ch, 42)
+}
+
+type InterfaceExample interface {
+	f1(arg int) string
+	f2(arg string) int
+	MySuperType
+}
+
+type MySuperType interface {
+	superMethod()
+}
+
+type Implementation struct {}
+
+func (imp Implementation) f1(arg int) string {
+	return "A"
+}
+
+func (imp Implementation) f2(arg string) int {
+	return 1
+}
+
+func typeAssertion(InterfaceExample i) {
+	fmt.Println(i.(Implementation))
 }