Add many more tokens, add tests and README

jplag.frontend.golang/README.md

@@ -0,0 +1,29 @@
+# JPlag Go language frontend
+
+The JPlag Go frontend allows the use of JPlag with submissions in Go. <br>
+It is based on the [Golang ANTLR4 grammar](https://github.com/antlr/grammars-v4/tree/master/golang), licensed under BSD-3.
+
+### Go specification compatibility
+
+The underlying grammar definition does not specify which version of Go it is built on. This is what we know:
+ - Number literal prefixes, a feature of go1.13, are included.
+ - Generics, a feature of go1.18, are _not_ included.
+ - Between go1.13 and go1.18, there were no changes to the syntax. So, the grammar should be fully compatible with go1.17, released in mid-2021.
+
+If the grammar is updated to a more recent<a href="#footnote-1"><sup>1</sup></a> syntax definition, this module should surely be updated as well.
+
+### Token Extraction
+
+The choice of tokens is intended to be similar to the Java or C# frontends. Specifically, among others, it includes a range of nesting structures (class and method declarations, control flow expressions) as well as variable declaration, object creation, assignment, and control flow altering keywords. <br>
+Blocks are distinguished by their context, i.e. there are separate `TokenConstants` for `if` blocks, `for` blocks, class bodies, method bodies, array constructors, and the like.
+
+More syntactic elements of Go may turn out to be helpful to include in the future, especially those that are newly introduced.
+
+### Usage
+
+To use the Go frontend, add the `-l golang` flag in the CLI, or use a `JPlagOption` object set to `LanguageOption.GO_LANG` in the Java API as described in the usage information in the [readme of the main project](https://github.com/jplag/JPlag#usage) and [in the wiki](https://github.com/jplag/JPlag/wiki/1.-How-to-Use-JPlag).
+
+<br>
+
+#### Footnotes
+<section id="footnote-1"><sup>1 </sup>Currently, the grammar is taken from grammar-v4, commit 51ecccf from April 2022.</section>

jplag.frontend.golang/pom.xml

@@ -18,6 +18,13 @@
             <groupId>org.antlr</groupId>
             <artifactId>antlr4-runtime</artifactId>
         </dependency>
+        <dependency>
+            <groupId>de.jplag</groupId>
+            <artifactId>frontend-testutils</artifactId>
+            <version>${revision}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

jplag.frontend.golang/src/main/java/de/jplag/golang/GoListener.java

@@ -3,6 +3,7 @@ package de.jplag.golang;
 import static de.jplag.golang.GoTokenConstants.*;
 
 import java.util.Arrays;
+import java.util.Optional;
 import java.util.Stack;
 
 import org.antlr.v4.runtime.Token;
@@ -13,6 +14,7 @@ import de.jplag.golang.grammar.GoParserBaseListener;
 
 public class GoListener extends GoParserBaseListener {
 
+    public static final int NONE = -1;
     private final GoParserAdapter parserAdapter;
     private final Stack<GoBlockContext> blockContexts;
 
@@ -50,8 +52,44 @@ public class GoListener extends GoParserBaseListener {
         assert Arrays.stream(contexts).anyMatch(context -> context == topContext);
     }
 
+    private GoBlockContext getCurrentContext() {
+        return blockContexts.peek();
+    }
+
     /* TOP LEVEL STRUCTURES */
 
+    @Override
+    public void enterPackageClause(GoParser.PackageClauseContext context) {
+        transformToken(PACKAGE, context.getStart(), context.getStop());
+        super.enterPackageClause(context);
+    }
+
+    @Override
+    public void enterImportDecl(GoParser.ImportDeclContext context) {
+        transformToken(IMPORT_DECL, context.getStart());
+
+        // if the children contain TerminalNodes, then it must be '(' and ')'
+        Optional<TerminalNode> listStart = context.children.stream().filter(tree -> tree instanceof TerminalNode).map(TerminalNode.class::cast)
+                .findFirst();
+        listStart.ifPresent(lParenTree -> transformToken(IMPORT_CLAUSE_BEGIN, lParenTree.getSymbol()));
+
+        super.enterImportDecl(context);
+    }
+
+    @Override
+    public void exitImportDecl(GoParser.ImportDeclContext context) {
+        if (context.getStop().getText().equals(")")) {
+            transformToken(IMPORT_CLAUSE_END, context.getStop());
+        }
+        super.exitImportDecl(context);
+    }
+
+    @Override
+    public void enterImportSpec(GoParser.ImportSpecContext context) {
+        transformToken(IMPORT_CLAUSE, context.getStart(), context.getStop());
+        super.enterImportSpec(context);
+    }
+
     /* STRUCT */
 
     @Override
@@ -107,20 +145,6 @@ public class GoListener extends GoParserBaseListener {
         super.enterParameterDecl(context);
     }
 
-    @Override
-    public void enterBlock(GoParser.BlockContext context) {
-        int tokenType = blockContexts.peek().getBegin();
-        transformToken(tokenType, context.getStart());
-        super.enterBlock(context);
-    }
-
-    @Override
-    public void exitBlock(GoParser.BlockContext context) {
-        int tokenType = blockContexts.peek().getEnd();
-        transformToken(tokenType, context.getStop());
-        super.enterBlock(context);
-    }
-
     /* CONTROL FLOW STATEMENTS */
 
     @Override
@@ -163,20 +187,92 @@ public class GoListener extends GoParserBaseListener {
     }
 
     @Override
-    public void enterExprSwitchCase(GoParser.ExprSwitchCaseContext context) {
+    public void enterExprCaseClause(GoParser.ExprCaseClauseContext context) {
         transformToken(SWITCH_CASE, context.getStart());
-        enterContext(GoBlockContext.CASE_BLOCK);
-        super.enterExprSwitchCase(context);
+        var caseBlock = context.getChild(GoParser.StatementListContext.class, 0);
+        if (caseBlock != null) {
+            enterContext(GoBlockContext.CASE_BLOCK);
+            transformToken(CASE_BLOCK_BEGIN, caseBlock.getStart());
+        }
+        super.enterExprCaseClause(context);
     }
 
     @Override
-    public void exitExprSwitchCase(GoParser.ExprSwitchCaseContext context) {
-        expectAndLeave(GoBlockContext.CASE_BLOCK);
-        super.exitExprSwitchCase(context);
+    public void exitExprCaseClause(GoParser.ExprCaseClauseContext context) {
+        if (getCurrentContext() == GoBlockContext.CASE_BLOCK) {
+            transformToken(CASE_BLOCK_END, context.getStop());
+            expectAndLeave(GoBlockContext.CASE_BLOCK);
+        }
+        super.exitExprCaseClause(context);
+    }
+
+    @Override
+    public void enterTypeCaseClause(GoParser.TypeCaseClauseContext context) {
+        transformToken(SWITCH_CASE, context.getStart());
+        var caseBlock = context.getChild(GoParser.StatementListContext.class, 0);
+        if (caseBlock != null) {
+            enterContext(GoBlockContext.CASE_BLOCK);
+            transformToken(CASE_BLOCK_BEGIN, caseBlock.getStart());
+        }
+        super.enterTypeCaseClause(context);
+    }
+
+    @Override
+    public void exitTypeCaseClause(GoParser.TypeCaseClauseContext context) {
+        if (getCurrentContext() == GoBlockContext.CASE_BLOCK) {
+            transformToken(CASE_BLOCK_END, context.getStop());
+            expectAndLeave(GoBlockContext.CASE_BLOCK);
+        }
+        super.exitTypeCaseClause(context);
+    }
+
+    @Override
+    public void enterSelectStmt(GoParser.SelectStmtContext context) {
+        transformToken(SELECT_STATEMENT, context.getStart());
+        enterContext(GoBlockContext.SELECT_CONTEXT);
+        super.enterSelectStmt(context);
+    }
+
+    @Override
+    public void exitSelectStmt(GoParser.SelectStmtContext context) {
+        expectAndLeave(GoBlockContext.SELECT_CONTEXT);
+        super.exitSelectStmt(context);
+    }
+
+    @Override
+    public void enterCommCase(GoParser.CommCaseContext context) {
+        transformToken(SWITCH_CASE, context.getStart());
+        var caseBlock = context.getChild(GoParser.StatementListContext.class, 0);
+        if (caseBlock != null) {
+            enterContext(GoBlockContext.CASE_BLOCK);
+            transformToken(CASE_BLOCK_BEGIN, caseBlock.getStart());
+        }
+        super.enterCommCase(context);
+    }
+
+    @Override
+    public void exitCommCase(GoParser.CommCaseContext context) {
+        if (getCurrentContext() == GoBlockContext.CASE_BLOCK) {
+            transformToken(CASE_BLOCK_END, context.getStop());
+            expectAndLeave(GoBlockContext.CASE_BLOCK);
+        }
+        super.exitCommCase(context);
     }
 
     /* STATEMENTS */
 
+    @Override
+    public void enterVarDecl(GoParser.VarDeclContext context) {
+        transformToken(VARIABLE_DECLARATION, context.getStart(), context.getStop());
+        super.enterVarDecl(context);
+    }
+
+    @Override
+    public void enterConstSpec(GoParser.ConstSpecContext context) {
+        transformToken(CONSTANT_DECLARATION, context.getStart());
+        super.enterConstSpec(context);
+    }
+
     @Override
     public void enterFunctionLit(GoParser.FunctionLitContext context) {
         transformToken(FUNCTION_LITERAL, context.getStart());
@@ -196,16 +292,21 @@ public class GoListener extends GoParserBaseListener {
         super.enterAssignment(context);
     }
 
+    @Override
+    public void enterShortVarDecl(GoParser.ShortVarDeclContext context) {
+        transformToken(VARIABLE_DECLARATION, context.getStart());
+        transformToken(ASSIGNMENT, context.getStart());
+        super.enterShortVarDecl(context);
+    }
+
     @Override
     public void enterArguments(GoParser.ArgumentsContext context) {
         transformToken(INVOCATION, context.getStart(), context.getStop());
 
         // Arguments consist of ExpressionLists, which consist of Expressions
         // Get all Expressions of all ExpressionLists in this ArgumentsContext
-        context.children.stream().filter(child -> child instanceof GoParser.ExpressionListContext)
-                .map(child -> (GoParser.ExpressionListContext) child)
-                .flatMap(child -> child.children.stream()).filter(child -> child instanceof GoParser.ExpressionContext)
-                .map(child -> (GoParser.ExpressionContext) child)
+        context.getRuleContexts(GoParser.ExpressionListContext.class).stream()
+                .flatMap(child -> child.getRuleContexts(GoParser.ExpressionContext.class).stream())
                 .forEachOrdered(arg -> transformToken(ARGUMENT, arg.getStart(), arg.getStop()));
         super.enterArguments(context);
     }
@@ -224,36 +325,59 @@ public class GoListener extends GoParserBaseListener {
 
     /* OBJECT CREATION */
 
-    @Override
-    public void enterCompositeLit(GoParser.CompositeLitContext context) {
-        transformToken(STRUCT_CONSTRUCTOR, context.getStart());
-        super.enterCompositeLit(context);
-    }
-
     @Override
     public void enterKeyedElement(GoParser.KeyedElementContext context) {
-        transformToken(STRUCT_VALUE, context.getStart(), context.getStop());
+        int tokenType = getCurrentContext().getElement();
+        transformToken(tokenType, context.getStart(), context.getStop());
+
         super.enterKeyedElement(context);
     }
 
     @Override
     public void enterArrayType(GoParser.ArrayTypeContext context) {
-        transformToken(ARRAY_CONSTRUCTOR, context.getStart(), context.getStop());
+        // otherwise, it is just a type expression
+        if (context.parent.parent instanceof GoParser.CompositeLitContext) {
+            enterContext(GoBlockContext.ARRAY_BODY);
+            transformToken(ARRAY_CONSTRUCTOR, context.getStart(), context.getStop());
+        }
         super.enterArrayType(context);
     }
 
     @Override
     public void enterSliceType(GoParser.SliceTypeContext context) {
-        transformToken(SLICE_CONSTRUCTOR, context.getStart(), context.getStop());
+        // otherwise, it is just a type expression
+        if (context.parent.parent instanceof GoParser.CompositeLitContext) {
+            enterContext(GoBlockContext.SLICE_BODY);
+            transformToken(SLICE_CONSTRUCTOR, context.getStart(), context.getStop());
+        }
         super.enterSliceType(context);
     }
 
+    @Override
+    public void exitCompositeLit(GoParser.CompositeLitContext context) {
+        expectAndLeave(GoBlockContext.MAP_BODY, GoBlockContext.SLICE_BODY, GoBlockContext.ARRAY_BODY, GoBlockContext.NAMED_TYPE_BODY);
+        super.exitCompositeLit(context);
+    }
+
     @Override
     public void enterMapType(GoParser.MapTypeContext context) {
-        transformToken(MAP_CONSTRUCTOR, context.getStart(), context.getStop());
+        // otherwise, it is just a type expression
+        if (context.parent.parent instanceof GoParser.CompositeLitContext) {
+            enterContext(GoBlockContext.MAP_BODY);
+            transformToken(MAP_CONSTRUCTOR, context.getStart(), context.getStop());
+        }
         super.enterMapType(context);
     }
 
+    @Override
+    public void enterTypeName(GoParser.TypeNameContext context) {
+        if (context.parent.parent instanceof GoParser.CompositeLitContext) {
+            transformToken(NAMED_TYPE_CONSTRUCTOR, context.getStart());
+            enterContext(GoBlockContext.NAMED_TYPE_BODY);
+        }
+        super.enterTypeName(context);
+    }
+
     /* CONTROL FLOW KEYWORDS */
 
     @Override
@@ -274,6 +398,12 @@ public class GoListener extends GoParserBaseListener {
         super.enterContinueStmt(context);
     }
 
+    @Override
+    public void enterFallthroughStmt(GoParser.FallthroughStmtContext context) {
+        transformToken(FALLTHROUGH, context.getStart(), context.getStop());
+        super.enterFallthroughStmt(context);
+    }
+
     @Override
     public void enterGotoStmt(GoParser.GotoStmtContext context) {
         transformToken(GOTO, context.getStart(), context.getStop());
@@ -292,31 +422,56 @@ public class GoListener extends GoParserBaseListener {
         super.enterDeferStmt(context);
     }
 
+    @Override
+    public void enterSendStmt(GoParser.SendStmtContext ctx) {
+        transformToken(SEND_STATEMENT, ctx.getStart(), ctx.getStop());
+        super.enterSendStmt(ctx);
+    }
+
+    @Override
+    public void enterRecvStmt(GoParser.RecvStmtContext ctx) {
+        transformToken(RECEIVE_STATEMENT, ctx.getStart(), ctx.getStop());
+        super.enterRecvStmt(ctx);
+    }
+
     @Override
     public void visitTerminal(TerminalNode node) {
-        if (node.getSymbol().getText().equals("else")) {
-            expectAndLeave(GoBlockContext.IF_BLOCK);
-            enterContext(GoBlockContext.ELSE_BLOCK);
+        Token token = node.getSymbol();
+        switch (token.getText()) {
+            case "else" -> {
+                expectAndLeave(GoBlockContext.IF_BLOCK);
+                enterContext(GoBlockContext.ELSE_BLOCK);
+            }
+            case "{" -> transformToken(getCurrentContext().getBegin(), token);
+            case "}" -> transformToken(getCurrentContext().getEnd(), token);
         }
         super.visitTerminal(node);
     }
 
     private enum GoBlockContext {
-        STRUCT_BODY(STRUCT_BODY_BEGIN, STRUCT_BODY_END),
-        FUNCTION_BODY(FUNCTION_BODY_BEGIN, FUNCTION_BODY_END),
-        IF_BLOCK(IF_BLOCK_BEGIN, IF_BLOCK_END),
-        ELSE_BLOCK(ELSE_BLOCK_BEGIN, ELSE_BLOCK_END),
-        FOR_BLOCK(FOR_BLOCK_BEGIN, FOR_BLOCK_END),
-        SWITCH_BLOCK(SWITCH_BLOCK_BEGIN, SWITCH_BLOCK_END),
-        CASE_BLOCK(CASE_BLOCK_BEGIN, CASE_BLOCK_END),
-        STATEMENT_BLOCK(STATEMENT_BLOCK_BEGIN, STATEMENT_BLOCK_END);
+        ARRAY_BODY(ARRAY_BODY_BEGIN, ARRAY_BODY_END, ARRAY_ELEMENT),
+        STRUCT_BODY(STRUCT_BODY_BEGIN, STRUCT_BODY_END, MEMBER_DECLARATION),
+        MAP_BODY(MAP_BODY_BEGIN, MAP_BODY_END, MAP_ELEMENT),
+        SLICE_BODY(SLICE_BODY_BEGIN, SLICE_BODY_END, SLICE_ELEMENT),
+        NAMED_TYPE_BODY(NAMED_TYPE_BODY_BEGIN, NAMED_TYPE_BODY_END, NAMED_TYPE_ELEMENT),
+        FUNCTION_BODY(FUNCTION_BODY_BEGIN, FUNCTION_BODY_END, NONE),
+
+        IF_BLOCK(IF_BLOCK_BEGIN, IF_BLOCK_END, NONE),
+        ELSE_BLOCK(ELSE_BLOCK_BEGIN, ELSE_BLOCK_END, NONE),
+        FOR_BLOCK(FOR_BLOCK_BEGIN, FOR_BLOCK_END, NONE),
+        SWITCH_BLOCK(SWITCH_BLOCK_BEGIN, SWITCH_BLOCK_END, NONE),
+        SELECT_CONTEXT(SELECT_BLOCK_BEGIN, SELECT_BLOCK_END, NONE),
+        STATEMENT_BLOCK(STATEMENT_BLOCK_BEGIN, STATEMENT_BLOCK_END, NONE),
+        CASE_BLOCK(CASE_BLOCK_BEGIN, CASE_BLOCK_END, NONE);
 
         private final int beginTokenType;
         private final int endTokenType;
+        private final int elementTokenType;
 
-        GoBlockContext(int beginTokenType, int endTokenType) {
+        GoBlockContext(int beginTokenType, int endTokenType, int elementTokenType) {
             this.beginTokenType = beginTokenType;
             this.endTokenType = endTokenType;
+            this.elementTokenType = elementTokenType;
         }
 
         int getBegin() {
@@ -326,5 +481,9 @@ public class GoListener extends GoParserBaseListener {
         int getEnd() {
             return this.endTokenType;
         }
+
+        public int getElement() {
+            return this.elementTokenType;
+        }
     }
 }

jplag.frontend.golang/src/main/java/de/jplag/golang/GoToken.java

@@ -16,10 +16,28 @@ public class GoToken extends Token {
             case FILE_END -> "<EOF>";
             case SEPARATOR_TOKEN -> "---------";
 
+            case PACKAGE -> "PACKAGE";
+            case IMPORT_CLAUSE -> "IMPORT";
+            case IMPORT_CLAUSE_BEGIN -> "IMPORT(";
+            case IMPORT_CLAUSE_END -> ")IMPORT";
+            case IMPORT_DECL -> "IMPORT_DECL";
+
+            case ARRAY_BODY_BEGIN -> "ARRAY{";
+            case ARRAY_BODY_END -> "}ARRAY";
+
             case STRUCT_DECLARATION_BEGIN -> "STRUCT";
             case STRUCT_BODY_BEGIN -> "STRUCT{";
             case STRUCT_BODY_END -> "}STRUCT";
 
+            case MAP_BODY_BEGIN -> "MAP{";
+            case MAP_BODY_END -> "}MAP";
+
+            case SLICE_BODY_BEGIN -> "SLICE{";
+            case SLICE_BODY_END -> "}SLICE";
+
+            case NAMED_TYPE_BODY_BEGIN -> "NAMED_TYPE{";
+            case NAMED_TYPE_BODY_END -> "}NAMED_TYPE";
+
             case MEMBER_DECLARATION -> "FIELD";
 
             case FUNCTION_DECLARATION -> "FUNC";
@@ -43,30 +61,42 @@ public class GoToken extends Token {
             case SWITCH_BLOCK_BEGIN -> "SWITCH{";
             case SWITCH_BLOCK_END -> "}SWITCH";
             case SWITCH_CASE -> "CASE";
+            case SELECT_STATEMENT -> "SELECT";
+            case SELECT_BLOCK_BEGIN -> "SELECT{";
+            case SELECT_BLOCK_END -> "}SELECT";
             case CASE_BLOCK_BEGIN -> "CASE{";
             case CASE_BLOCK_END -> "}CASE";
 
             // Statements
 
+            case VARIABLE_DECLARATION -> "VAR_DECL";
+            case CONSTANT_DECLARATION -> "CONST_DECL";
             case FUNCTION_LITERAL -> "FUNC_LIT";
             case ASSIGNMENT -> "ASSIGN";
+            case SEND_STATEMENT -> "SEND";
+            case RECEIVE_STATEMENT -> "RECV";
             case INVOCATION -> "INVOC";
             case ARGUMENT -> "ARG";
             case STATEMENT_BLOCK_BEGIN -> "INNER{";
             case STATEMENT_BLOCK_END -> "}INNER";
 
             // Object Creation
-            case STRUCT_CONSTRUCTOR -> "STRUCT()";
-            case STRUCT_VALUE -> "ARG";
+
+            case ARRAY_ELEMENT -> "ARRAY_ELEM";
+            case MAP_ELEMENT -> "MAP_ELEM";
+            case SLICE_ELEMENT -> "SLICE_ELEM";
+            case NAMED_TYPE_ELEMENT -> "CTYPE_ELEM";
             case ARRAY_CONSTRUCTOR -> "ARRAY()";
             case SLICE_CONSTRUCTOR -> "SLICE()";
             case MAP_CONSTRUCTOR -> "MAP()";
+            case NAMED_TYPE_CONSTRUCTOR -> "CTYPE()";
 
             // Control Flow Keywords
 
             case RETURN -> "RETURN";
             case BREAK -> "BREAK";
             case CONTINUE -> "CONTINUE";
+            case FALLTHROUGH -> "FALLTHROUGH";
             case GOTO -> "GOTO";
             case GO -> "GO";
             case DEFER -> "DEFER";

jplag.frontend.golang/src/main/java/de/jplag/golang/GoTokenConstants.java

@@ -6,59 +6,83 @@ public interface GoTokenConstants extends TokenConstants {
 
     // Top level structures
 
-    int STRUCT_DECLARATION_BEGIN = 2;
-    int STRUCT_BODY_BEGIN = 3;
-    int STRUCT_BODY_END = 4;
-    int MEMBER_DECLARATION = 5;
-    int FUNCTION_DECLARATION = 6;
-    int METHOD_DECLARATION = 7;
-    int FUNCTION_PARAMETER = 8;
-    int FUNCTION_BODY_BEGIN = 9;
-    int FUNCTION_BODY_END = 10;
+    int PACKAGE = 2;
+    int IMPORT_CLAUSE = 3;
+    int IMPORT_CLAUSE_BEGIN = 4;
+    int IMPORT_CLAUSE_END = 5;
+    int IMPORT_DECL = 6;
+    int ARRAY_BODY_BEGIN = 7;
+    int ARRAY_BODY_END = 8;
+    int STRUCT_DECLARATION_BEGIN = 9;
+    int STRUCT_BODY_BEGIN = 10;
+    int STRUCT_BODY_END = 11;
+    int MAP_BODY_BEGIN = 12;
+    int MAP_BODY_END = 13;
+    int SLICE_BODY_BEGIN = 14;
+    int SLICE_BODY_END = 15;
+    int NAMED_TYPE_BODY_BEGIN = 16;
+    int NAMED_TYPE_BODY_END = 17;
+    int MEMBER_DECLARATION = 18;
+    int FUNCTION_DECLARATION = 19;
+    int METHOD_DECLARATION = 20;
+    int FUNCTION_PARAMETER = 21;
+    int FUNCTION_BODY_BEGIN = 22;
+    int FUNCTION_BODY_END = 23;
 
     // Control flow statements
 
-    int IF_STATEMENT = 11;
-    int IF_BLOCK_BEGIN = 12;
-    int IF_BLOCK_END = 13;
-    int ELSE_BLOCK_BEGIN = 14;
-    int ELSE_BLOCK_END = 15;
-    int FOR_STATEMENT = 16;
-    int FOR_BLOCK_BEGIN = 17;
-    int FOR_BLOCK_END = 18;
-    int SWITCH_STATEMENT = 19;
-    int SWITCH_BLOCK_BEGIN = 20;
-    int SWITCH_BLOCK_END = 21;
-    int SWITCH_CASE = 22;
-    int CASE_BLOCK_BEGIN = 23;
-    int CASE_BLOCK_END = 24;
+    int IF_STATEMENT = 24;
+    int IF_BLOCK_BEGIN = 25;
+    int IF_BLOCK_END = 26;
+    int ELSE_BLOCK_BEGIN = 27;
+    int ELSE_BLOCK_END = 28;
+    int FOR_STATEMENT = 29;
+    int FOR_BLOCK_BEGIN = 30;
+    int FOR_BLOCK_END = 31;
+    int SWITCH_STATEMENT = 32;
+    int SWITCH_BLOCK_BEGIN = 33;
+    int SWITCH_BLOCK_END = 34;
+    int SWITCH_CASE = 35;
+    int SELECT_STATEMENT = 36;
+    int SELECT_BLOCK_BEGIN = 37;
+    int SELECT_BLOCK_END = 38;
+    int CASE_BLOCK_BEGIN = 39;
+    int CASE_BLOCK_END = 40;
 
     // Statements
 
-    int FUNCTION_LITERAL = 25;
-    int ASSIGNMENT = 26;
-    int INVOCATION = 27;
-    int ARGUMENT = 28;
-    int STATEMENT_BLOCK_BEGIN = 29;
-    int STATEMENT_BLOCK_END = 30;
+    int VARIABLE_DECLARATION = 41;
+    int CONSTANT_DECLARATION = 42;
+    int FUNCTION_LITERAL = 43;
+    int ASSIGNMENT = 44;
+    int SEND_STATEMENT = 45;
+    int RECEIVE_STATEMENT = 46;
+    int INVOCATION = 47;
+    int ARGUMENT = 48;
+    int STATEMENT_BLOCK_BEGIN = 49;
+    int STATEMENT_BLOCK_END = 50;
 
-    // Object Creation
+    // Object creation
 
-    int STRUCT_CONSTRUCTOR = 31;
-    int STRUCT_VALUE = 32;
-    int ARRAY_CONSTRUCTOR = 33;
-    int SLICE_CONSTRUCTOR = 34;
-    int MAP_CONSTRUCTOR = 35;
+    int ARRAY_ELEMENT = 51;
+    int MAP_ELEMENT = 52;
+    int SLICE_ELEMENT = 53;
+    int NAMED_TYPE_ELEMENT = 54;
+    int ARRAY_CONSTRUCTOR = 55;
+    int MAP_CONSTRUCTOR = 56;
+    int SLICE_CONSTRUCTOR = 57;
+    int NAMED_TYPE_CONSTRUCTOR = 58;
 
-    // Control Flow Keywords
+    // Control flow keywords
 
-    int RETURN = 36;
-    int BREAK = 37;
-    int CONTINUE = 38;
-    int GOTO = 39;
-    int GO = 40;
-    int DEFER = 41;
+    int RETURN = 59;
+    int BREAK = 60;
+    int CONTINUE = 61;
+    int FALLTHROUGH = 62;
+    int GOTO = 63;
+    int GO = 64;
+    int DEFER = 65;
 
-    int NUM_DIFF_TOKENS = 42;
+    int NUM_DIFF_TOKENS = 66;
 
 }

jplag.frontend.golang/src/main/java/de/jplag/golang/Language.java

@@ -10,6 +10,7 @@ public class Language implements de.jplag.Language {
     private static final String SHORT_NAME = "Go";
     private static final String NAME = "Go Parser";
     private static final int DEFAULT_MIN_TOKEN_MATCH = 8;
+    private static final String[] FILE_EXTENSIONS = {".go"};
     private final GoParserAdapter parserAdapter;
 
     public Language(ErrorConsumer consumer) {
@@ -18,7 +19,7 @@ public class Language implements de.jplag.Language {
 
     @Override
     public String[] suffixes() {
-        return new String[] {".go"};
+        return FILE_EXTENSIONS;
     }
 
     @Override

jplag.frontend.golang/src/test/java/de/jplag/golang/GoFrontendTest.java

@@ -0,0 +1,176 @@
+package de.jplag.golang;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.StreamSupport;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import de.jplag.Token;
+import de.jplag.TokenConstants;
+import de.jplag.TokenList;
+import de.jplag.TokenPrinter;
+import de.jplag.testutils.TestErrorConsumer;
+
+public class GoFrontendTest {
+    /**
+     * Test source file that is supposed to produce a complete set of tokens, i.e. all types of tokens.
+     */
+    private static final String COMPLETE_TEST_FILE = "Complete.go";
+
+    /**
+     * Regular expression that describes lines consisting only of whitespace and optionally a line comment.
+     */
+    private static final String EMPTY_OR_SINGLE_LINE_COMMENT = "\\s*(//.*|/\\*.*\\*/)?";
+
+    /**
+     * Regular expression that describes lines containing the start of a multiline comment and no code before it.
+     */
+    private static final String DELIMITED_COMMENT_START = "\\s*/\\*(?:(?!\\*/).)*$";
+
+    /**
+     * Regular expression that describes lines containing the end of a multiline comment and no more code after that.
+     */
+    private static final String DELIMITED_COMMENT_END = ".*\\*/\\s*$";
+    private static final String NOT_SET_STRING = "";
+    private static final int NOT_SET = -1;
+
+    private final Logger logger = LoggerFactory.getLogger("GoLang frontend test");
+    private final String[] testFiles = new String[] {COMPLETE_TEST_FILE};
+    private final File testFileLocation = Path.of("src", "test", "resources", "de", "jplag", "golang").toFile();
+    private Language language;
+
+    @BeforeEach
+    void setup() {
+        TestErrorConsumer consumer = new TestErrorConsumer();
+        language = new Language(consumer);
+    }
+
+    @Test
+    void parseTestFiles() {
+        for (String fileName : testFiles) {
+            TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
+            String output = TokenPrinter.printTokens(tokens, testFileLocation, List.of(fileName));
+            logger.info(output);
+
+            testSourceCoverage(fileName, tokens);
+            if (fileName.equals(COMPLETE_TEST_FILE)) {
+                testTokenCoverage(tokens, fileName);
+            }
+        }
+    }
+
+    /**
+     * Confirms that every type of GoToken has a Sting representation associated to it.
+     */
+    @Test
+    void testToken2String() {
+        var missingTokens = IntStream.range(0, language.numberOfTokens())
+                .mapToObj(type -> new GoToken(type, NOT_SET_STRING, NOT_SET, NOT_SET, NOT_SET))
+                .filter(token -> token.type2string().contains("UNKNOWN")).toList();
+
+        if (!missingTokens.isEmpty()) {
+            var typeList = missingTokens.stream().map(Token::getType).map(Object::toString).collect(Collectors.joining(", "));
+            fail("Found token types with no string representation: %s".formatted(typeList));
+        }
+
+    }
+
+    /**
+     * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
+     * @param fileName a code sample file name
+     * @param tokens the TokenList generated from the sample
+     */
+    private void testSourceCoverage(String fileName, TokenList tokens) {
+        File testFile = new File(testFileLocation, fileName);
+
+        try {
+            List<String> lines = Files.readAllLines(testFile.toPath());
+
+            // All lines that contain code
+            var codeLines = getCodeLines(lines);
+            // All lines that contain token
+            var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().boxed().toList();
+
+            if (codeLines.size() > tokenLines.size()) {
+                List<Integer> missedLinesIndices = new ArrayList<>(codeLines);
+                missedLinesIndices.removeAll(tokenLines);
+                var missedLines = missedLinesIndices.stream().filter(index -> !lines.get(index - 1).matches(".*(\\+\\+|--).*")).map(Object::toString)
+                        .collect(Collectors.joining(", "));
+                if (!missedLines.isBlank()) {
+                    fail("Found lines in file '%s' that are not represented in the token list. \n\tMissed lines: %s".formatted(fileName,
+                            missedLines));
+                }
+            }
+            IntStream.range(0, codeLines.size()).dropWhile(index -> Objects.equals(codeLines.get(index), tokenLines.get(index))).findAny()
+                    .ifPresent(i -> fail(
+                            "Not all lines of code in '%s' are represented in tokens, starting with line %d.".formatted(fileName, codeLines.get(i))));
+        } catch (IOException exception) {
+            logger.info("Error while reading test file %s".formatted(fileName), exception);
+            fail();
+        }
+    }
+
+    /**
+     * Gets the line numbers of lines containing actual code, omitting empty lines and comment lines.
+     * @param lines lines of a code file
+     * @return an array of the line numbers of code lines
+     */
+    private List<Integer> getCodeLines(List<String> lines) {
+        // This boxed boolean can be accessed from within the lambda method below
+        var state = new Object() {
+            boolean insideComment = false;
+        };
+
+        var codeLines = IntStream.rangeClosed(1, lines.size()).sequential().filter(idx -> {
+            String line = lines.get(idx - 1);
+            if (line.matches(EMPTY_OR_SINGLE_LINE_COMMENT)) {
+                return false;
+            } else if (line.matches(DELIMITED_COMMENT_START)) {
+                state.insideComment = true;
+                return false;
+            } else if (state.insideComment) {
+                // This fails if code follows after '*/'. If the code is formatted well, this should not happen.
+                if (line.matches(DELIMITED_COMMENT_END)) {
+                    state.insideComment = false;
+                }
+                return false;
+            }
+            return true;
+        });
+
+        return codeLines.boxed().toList();
+
+    }
+
+    /**
+     * Confirms that all Token types are 'reachable' with a complete code example.
+     * @param tokens TokenList which is supposed to contain all types of tokens
+     * @param fileName The file name of the complete code example
+     */
+    private void testTokenCoverage(TokenList tokens, String fileName) {
+        var foundTokens = StreamSupport.stream(tokens.allTokens().spliterator(), true).mapToInt(Token::getType).sorted().distinct().toArray();
+        // Exclude SEPARATOR_TOKEN, as it does not occur
+        var allTokens = IntStream.range(0, language.numberOfTokens()).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).toArray();
+
+        if (allTokens.length > foundTokens.length) {
+            var diffLine = IntStream.range(0, allTokens.length)
+                    .dropWhile(lineIdx -> lineIdx < foundTokens.length && allTokens[lineIdx] == foundTokens[lineIdx]).findFirst();
+            diffLine.ifPresent(lineIdx -> fail("Token type %s was not found in the complete code example '%s'."
+                    .formatted(new GoToken(allTokens[lineIdx], fileName, -1, -1, -1).type2string(), fileName)));
+        }
+        assertArrayEquals(allTokens, foundTokens);
+    }
+}

jplag.frontend.golang/src/test/resources/de/jplag/golang/Complete.go

@@ -0,0 +1,143 @@
+package goExample
+
+import (
+	"fmt"
+	"strings"
+)
+
+type myStruct struct {
+	int number
+	string name
+
+	int	[]data
+}
+
+func (this myStruct) init(number int, name string, data []int) {
+	this.number = number
+	this.name = name
+	this.data = data
+}
+
+func (this myStruct) print() {
+	const (intro = "This is %s, myStruct number %d, which contains %d elements."; separator = "-------------------")
+
+	fmt.Printf(intro, this.name, this.number, len(this.data))
+	fmt.Println(separator)
+}
+
+func getNth(structs []myStruct, index i) myStruct {
+	if i < len(structs) {
+		return structs[i]
+	}
+
+	return nil
+}
+
+func printAll(structs []myStruct) {
+	for i := range structs {
+		var element = getNth(structs, i)
+		element.print()
+	}
+}
+
+func deferredFunction() {
+	fmt.println("This is printed even after panic.")
+}
+
+func panicExample() {
+	defer defferedFunction()
+	panic("This operation does not work!")
+}
+
+func switchExample(int dividend, int divisor) {
+	switch {
+		case divisor == 1: return dividend
+		case divisor == 0: panicExample()
+		case dividend == 0: return 0
+		default: return dividend / divisor
+	}
+}
+
+var isEven = map[int]bool{
+	0: true,
+	1: false,
+	2: true,
+	3: false,
+}
+
+func checkIsEven(value int) bool {
+	f := func(value int) bool {
+		if value >= 0 && value <= 3 {
+			return isEven[value]
+		} else {
+			return value % 2 == 0
+		}
+	}
+
+	return f(value)
+}
+
+func sliceExample() []int {
+	return []int{1, 2, 3}
+}
+
+func fallthroughExample(value int) {
+	switch {
+		case value > 10:
+			fmt.Println("Big number!")
+			fallthrough
+		case value > 100:
+			fmt.Println("Very big number!")
+	}
+}
+
+func createCType() {
+	var element = myStruct{
+		number: 1,
+		name: "myElement",
+		data: [5]int{1, 2, 3, 4, 5},
+	}
+
+}
+
+func createArray() *ast.File {
+	var array = [3]int { nil, nil, nil }
+	return array
+}
+
+func selectExample() int {
+	select {
+		case signal1 := <-ch1:
+			fmt.Println("Haha! ", signal1)
+			return signal1
+		case signal2 := <-ch2:
+			return signal2
+		default: return -1
+	}
+}
+
+func innerBlock() {
+	fmt.Println("Outer")
+	{
+		fmt.Println("Inner")
+	}
+}
+
+func continueExample() {
+	begin: for true {
+		if 1 > 0 {
+			continue;
+		}
+		break;
+	}
+	goto begin
+}
+
+func channelExample(ch chan<- int, value int) {
+	ch <- value
+}
+
+func goExample() {
+	ch := make(chan<- int)
+	go innerBlock(ch, 42)
+}