Merge branch 'master' into refactoring/languages

# Conflicts:
#	language-api/src/main/java/de/jplag/SharedTokenType.java
#	language-api/src/main/java/de/jplag/TokenConstants.java
#	language-api/src/main/java/de/jplag/TokenType.java
#	language-api/src/test/java/de/jplag/TestToken.java
#	languages/cpp/src/main/java/de/jplag/cpp/CPPToken.java
#	languages/cpp/src/main/java/de/jplag/cpp/CPPTokenConstants.java
#	languages/cpp/src/main/java/de/jplag/cpp/CPPTokenType.java
#	languages/csharp/src/main/java/de/jplag/csharp/CSharpToken.java
#	languages/csharp/src/main/java/de/jplag/csharp/CSharpTokenConstants.java
#	languages/csharp/src/main/java/de/jplag/csharp/CSharpTokenType.java
#	languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/DynamicMetamodelTokenConstants.java
#	languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/DynamicMetamodelTokenType.java
#	languages/emf-metamodel/src/main/java/de/jplag/emf/MetamodelTokenConstants.java
#	languages/emf-metamodel/src/main/java/de/jplag/emf/MetamodelTokenType.java
#	languages/golang/src/main/java/de/jplag/golang/GoToken.java
#	languages/golang/src/main/java/de/jplag/golang/GoTokenConstants.java
#	languages/golang/src/main/java/de/jplag/golang/GoTokenType.java
#	languages/golang/src/test/java/de/jplag/golang/GoTokenTestUtils.java
#	languages/java/src/main/java/de/jplag/java/JavaToken.java
#	languages/java/src/main/java/de/jplag/java/JavaTokenConstants.java
#	languages/java/src/main/java/de/jplag/java/JavaTokenType.java
#	languages/kotlin/src/main/java/de/jplag/kotlin/KotlinToken.java
#	languages/kotlin/src/main/java/de/jplag/kotlin/KotlinTokenConstants.java
#	languages/kotlin/src/main/java/de/jplag/kotlin/KotlinTokenType.java
#	languages/python-3/src/main/java/de/jplag/python3/Python3Token.java
#	languages/python-3/src/main/java/de/jplag/python3/Python3TokenConstants.java
#	languages/python-3/src/main/java/de/jplag/python3/Python3TokenType.java
#	languages/rlang/src/main/java/de/jplag/rlang/RToken.java
#	languages/rlang/src/main/java/de/jplag/rlang/RTokenConstants.java
#	languages/rlang/src/main/java/de/jplag/rlang/RTokenType.java
#	languages/rust/src/main/java/de/jplag/rust/RustToken.java
#	languages/rust/src/main/java/de/jplag/rust/RustTokenConstants.java
#	languages/rust/src/main/java/de/jplag/rust/RustTokenType.java
#	languages/scala/src/main/scala/de/jplag/scala/ScalaToken.scala
#	languages/scala/src/main/scala/de/jplag/scala/ScalaTokenConstants.scala
#	languages/scala/src/main/scala/de/jplag/scala/ScalaTokenType.java
#	languages/scheme/src/main/java/de/jplag/scheme/SchemeToken.java
#	languages/scheme/src/main/java/de/jplag/scheme/SchemeTokenConstants.java
#	languages/scheme/src/main/java/de/jplag/scheme/SchemeTokenType.java
#	languages/text/src/main/java/de/jplag/text/TextToken.java
#	languages/text/src/main/java/de/jplag/text/TextTokenType.java

core/src/main/java/de/jplag/GreedyStringTiling.java

@@ -1,21 +1,22 @@
 package de.jplag;
 
-import static de.jplag.TokenConstants.FILE_END;
-import static de.jplag.TokenConstants.SEPARATOR_TOKEN;
-
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import de.jplag.options.JPlagOptions;
 
 /**
  * This class implements the Greedy String Tiling algorithm as introduced by Michael Wise. However, it is very specific
- * to the classes {@link Token}, and {@link Match}.
+ * to the classes {@link Token}, and {@link Match}. Class implementation is thread-safe, i.e. submission can be compared
+ * in parallel.
  * @see <a href=
  * "https://www.researchgate.net/publication/262763983_String_Similarity_via_Greedy_String_Tiling_and_Running_Karp-Rabin_Matching">
  * String Similarity via Greedy String Tiling and Running Karp−Rabin Matching </a>
@@ -23,11 +24,16 @@ import de.jplag.options.JPlagOptions;
 public class GreedyStringTiling {
 
     private final int minimumMatchLength;
-    private final Map<Submission, SubsequenceHashLookupTable> cachedHashLookupTables = new IdentityHashMap<>();
+    private ConcurrentMap<TokenType, Integer> tokenTypeValues;
     private final Map<Submission, Set<Token>> baseCodeMarkings = new IdentityHashMap<>();
 
+    private final Map<Submission, int[]> cachedTokenValueLists = new IdentityHashMap<>();
+    private final Map<Submission, SubsequenceHashLookupTable> cachedHashLookupTables = new IdentityHashMap<>();
+
     public GreedyStringTiling(JPlagOptions options) {
         this.minimumMatchLength = options.minimumTokenMatch();
+        this.tokenTypeValues = new ConcurrentHashMap<>();
+        this.tokenTypeValues.put(SharedTokenType.FILE_END, 0);
     }
 
     /**
@@ -89,54 +95,48 @@ public class GreedyStringTiling {
         List<Token> leftTokens = leftSubmission.getTokenList();
         List<Token> rightTokens = rightSubmission.getTokenList();
 
+        int[] leftValues = tokenValueListFromSubmission(leftSubmission);
+        int[] rightValues = tokenValueListFromSubmission(rightSubmission);
+
         // comparison uses <= because it is assumed that the last token is a pivot (FILE_END)
         if (leftTokens.size() <= minimumMatchLength || rightTokens.size() <= minimumMatchLength) {
             return new JPlagComparison(leftSubmission, rightSubmission, List.of());
         }
 
-        Set<Token> leftMarkedTokens = initiallyMarkedTokens(leftSubmission);
-        Set<Token> rightMarkedTokens = initiallyMarkedTokens(rightSubmission);
+        Set<Integer> leftMarkedIndexes = initiallyMarkedTokenIndexes(leftSubmission);
+        Set<Integer> rightMarkedIndexes = initiallyMarkedTokenIndexes(rightSubmission);
 
-        SubsequenceHashLookupTable leftLookupTable = subsequenceHashLookupTableForSubmission(leftSubmission, leftMarkedTokens);
-        SubsequenceHashLookupTable rightLookupTable = subsequenceHashLookupTableForSubmission(rightSubmission, rightMarkedTokens);
+        SubsequenceHashLookupTable leftLookupTable = subsequenceHashLookupTableForSubmission(leftSubmission, leftMarkedIndexes);
+        SubsequenceHashLookupTable rightLookupTable = subsequenceHashLookupTableForSubmission(rightSubmission, rightMarkedIndexes);
 
         int maximumMatchLength;
         List<Match> globalMatches = new ArrayList<>();
         do {
             maximumMatchLength = minimumMatchLength;
             List<Match> iterationMatches = new ArrayList<>();
-            for (int leftStartIndex = 0; leftStartIndex < leftTokens.size() - maximumMatchLength; leftStartIndex++) {
+            for (int leftStartIndex = 0; leftStartIndex < leftValues.length - maximumMatchLength; leftStartIndex++) {
                 int leftSubsequenceHash = leftLookupTable.subsequenceHashForStartIndex(leftStartIndex);
-                if (leftMarkedTokens.contains(leftTokens.get(leftStartIndex)) || leftSubsequenceHash == SubsequenceHashLookupTable.NO_HASH) {
+                if (leftMarkedIndexes.contains(leftStartIndex) || leftSubsequenceHash == SubsequenceHashLookupTable.NO_HASH) {
                     continue;
                 }
                 List<Integer> possiblyMatchingRightStartIndexes = rightLookupTable
                         .startIndexesOfPossiblyMatchingSubsequencesForSubsequenceHash(leftSubsequenceHash);
                 for (Integer rightStartIndex : possiblyMatchingRightStartIndexes) {
                     // comparison uses >= because it is assumed that the last token is a pivot (FILE_END)
-                    if (rightMarkedTokens.contains(rightTokens.get(rightStartIndex)) || maximumMatchLength >= rightTokens.size() - rightStartIndex) {
+                    if (rightMarkedIndexes.contains(rightStartIndex) || maximumMatchLength >= rightValues.length - rightStartIndex) {
                         continue;
                     }
 
-                    if (!subsequencesAreMatchingAndNotMarked(leftTokens.subList(leftStartIndex, leftStartIndex + maximumMatchLength),
-                            leftMarkedTokens, rightTokens.subList(rightStartIndex, rightStartIndex + maximumMatchLength), rightMarkedTokens)) {
-                        continue;
-                    }
-
-                    // expand match
-                    int offset = maximumMatchLength;
-                    while (leftTokens.get(leftStartIndex + offset).type == rightTokens.get(rightStartIndex + offset).type
-                            && !leftMarkedTokens.contains(leftTokens.get(leftStartIndex + offset))
-                            && !rightMarkedTokens.contains(rightTokens.get(rightStartIndex + offset))) {
-                        offset++;
+                    int subsequenceMatchLength = maximalMatchingSubsequenceLengthNotMarked(leftValues, leftStartIndex, leftMarkedIndexes, rightValues,
+                            rightStartIndex, rightMarkedIndexes, maximumMatchLength);
+                    if (subsequenceMatchLength >= maximumMatchLength) {
+                        if (subsequenceMatchLength > maximumMatchLength) {
+                            iterationMatches.clear();
+                            maximumMatchLength = subsequenceMatchLength;
+                        }
+                        Match match = new Match(leftStartIndex, rightStartIndex, subsequenceMatchLength);
+                        addMatchIfNotOverlapping(iterationMatches, match);
                     }
-
-                    if (offset > maximumMatchLength) {
-                        iterationMatches.clear();
-                        maximumMatchLength = offset;
-                    }
-                    Match match = new Match(leftStartIndex, rightStartIndex, offset);
-                    addMatchIfNotOverlapping(iterationMatches, match);
                 }
             }
             for (Match match : iterationMatches) {
@@ -144,8 +144,8 @@ public class GreedyStringTiling {
                 int leftStartIndex = match.startOfFirst();
                 int rightStartIndex = match.startOfSecond();
                 for (int offset = 0; offset < match.length(); offset++) {
-                    leftMarkedTokens.add(leftTokens.get(leftStartIndex + offset));
-                    rightMarkedTokens.add(rightTokens.get(rightStartIndex + offset));
+                    leftMarkedIndexes.add(leftStartIndex + offset);
+                    rightMarkedIndexes.add(rightStartIndex + offset);
                 }
             }
         } while (maximumMatchLength != minimumMatchLength);
@@ -153,25 +153,35 @@ public class GreedyStringTiling {
     }
 
     /**
-     * Checks if the two provided subsequences are equal and not marked. Comparison is performed backwards based on the
-     * assumption that the further tokens are away, the more likely they differ. leftTokens and rightTokens must be of equal
-     * size.
-     * @param leftTokens The subsequence of left tokens.
-     * @param leftMarkedTokens The marked tokens of the left token list.
-     * @param rightTokens The subsequence of right tokens.
-     * @param rightMarkedTokens The marked tokens of the right token list.
-     * @return true if the subsequences are matching and not marked, otherwise false.
+     * Computes the maximal matching subsequence between the two lists starting at their respective indexes. Values are
+     * matching if they are equal and not marked. Comparison is performed backwards for the minimum sequence length based on
+     * the assumption that the further tokens are away, the more likely they differ.
+     * @param leftValues The list of left values.
+     * @param leftStartIndex The start index in the left list.
+     * @param leftMarkedIndexes The marked indexes of the left list.
+     * @param rightValues The list of right values.
+     * @param rightStartIndex The start index in the right list.
+     * @param rightMarkedIndexes The marked indexes of the right list.
+     * @param minimumSequenceLength The minimal sequence length for a matching subsequence. Must be not negative.
+     * @return the maximal matching subsequence length, or 0 if there is no subsequence of at least the minimum sequence
+     * length.
      */
-    private boolean subsequencesAreMatchingAndNotMarked(List<Token> leftTokens, Set<Token> leftMarkedTokens, List<Token> rightTokens,
-            Set<Token> rightMarkedTokens) {
-        for (int offset = leftTokens.size() - 1; offset >= 0; offset--) {
-            Token leftToken = leftTokens.get(offset);
-            Token rightToken = rightTokens.get(offset);
-            if (leftToken.type != rightToken.type || leftMarkedTokens.contains(leftToken) || rightMarkedTokens.contains(rightToken)) {
-                return false;
+    private int maximalMatchingSubsequenceLengthNotMarked(int[] leftValues, int leftStartIndex, Set<Integer> leftMarkedIndexes, int[] rightValues,
+            int rightStartIndex, Set<Integer> rightMarkedIndexes, int minimumSequenceLength) {
+        for (int offset = minimumSequenceLength - 1; offset >= 0; offset--) {
+            int leftIndex = leftStartIndex + offset;
+            int rightIndex = rightStartIndex + offset;
+            if (leftValues[leftIndex] != rightValues[rightIndex] || leftMarkedIndexes.contains(leftIndex)
+                    || rightMarkedIndexes.contains(rightIndex)) {
+                return 0;
             }
         }
-        return true;
+        int offset = minimumSequenceLength;
+        while (leftValues[leftStartIndex + offset] == rightValues[rightStartIndex + offset] && !leftMarkedIndexes.contains(leftStartIndex + offset)
+                && !rightMarkedIndexes.contains(rightStartIndex + offset)) {
+            offset++;
+        }
+        return offset;
     }
 
     private void addMatchIfNotOverlapping(List<Match> matches, Match match) {
@@ -183,19 +193,40 @@ public class GreedyStringTiling {
         matches.add(match);
     }
 
-    private Set<Token> initiallyMarkedTokens(Submission submission) {
+    private Set<Integer> initiallyMarkedTokenIndexes(Submission submission) {
         Set<Token> baseCodeTokens = baseCodeMarkings.get(submission);
-        return submission.getTokenList().stream().filter(
-                token -> token.type == FILE_END || token.type == SEPARATOR_TOKEN || (baseCodeTokens != null && baseCodeTokens.contains(token)))
-                .collect(Collectors.toSet());
+        List<Token> tokens = submission.getTokenList();
+        return IntStream.range(0, tokens.size())
+                .filter(i -> tokens.get(i).getType().isExcludedFromMatching() || (baseCodeTokens != null && baseCodeTokens.contains(tokens.get(i))))
+                .boxed().collect(Collectors.toSet());
     }
 
-    private SubsequenceHashLookupTable subsequenceHashLookupTableForSubmission(Submission submission, Set<Token> markedTokens) {
+    private SubsequenceHashLookupTable subsequenceHashLookupTableForSubmission(Submission submission, Set<Integer> markedIndexes) {
         if (cachedHashLookupTables.containsKey(submission)) {
             return cachedHashLookupTables.get(submission);
         }
-        SubsequenceHashLookupTable lookupTable = new SubsequenceHashLookupTable(minimumMatchLength, submission.getTokenList(), markedTokens);
+        SubsequenceHashLookupTable lookupTable = new SubsequenceHashLookupTable(minimumMatchLength, tokenValueListFromSubmission(submission),
+                markedIndexes);
         cachedHashLookupTables.put(submission, lookupTable);
         return lookupTable;
     }
+
+    /**
+     * Converts the tokens of the submission to a list of values.
+     * @param submission The submission from which to convert the tokens.
+     */
+    private int[] tokenValueListFromSubmission(Submission submission) {
+        if (cachedTokenValueLists.containsKey(submission)) {
+            return cachedTokenValueLists.get(submission);
+        }
+        List<Token> tokens = submission.getTokenList();
+        int[] tokenValueList = new int[tokens.size()];
+        for (int i = 0; i < tokens.size(); i++) {
+            TokenType type = tokens.get(i).getType();
+            tokenTypeValues.putIfAbsent(type, tokenTypeValues.size());
+            tokenValueList[i] = tokenTypeValues.get(type);
+        }
+        cachedTokenValueLists.put(submission, tokenValueList);
+        return tokenValueList;
+    }
 }

core/src/main/java/de/jplag/JPlagComparison.java

@@ -25,7 +25,7 @@ public record JPlagComparison(Submission firstSubmission, Submission secondSubmi
     /**
      * Get the total number of matched tokens for this comparison.
      */
-    public final int getNumberOfMatchedTokens() {
+    public int getNumberOfMatchedTokens() {
         return matches.stream().mapToInt(Match::length).sum();
     }
 

core/src/main/java/de/jplag/SubsequenceHashLookupTable.java

@@ -7,45 +7,45 @@ import java.util.Map;
 import java.util.Set;
 
 /**
- * A class to generate and store hashes over a fixed length subsequence of a given list of tokens. Hash generation is
+ * A class to generate and store hashes over a fixed length subsequence of a given list of values. Hash generation is
  * optimized to work in O(n).
  */
 class SubsequenceHashLookupTable {
     /**
      * Value combination is chosen such that the maximum possible hash value does not exceed Int.max. Computation formula
-     * for maximum hash value is \sum from (i=0 to MAX_HASH_LENGTH - 1) with (TOKEN_HASH_MODULO - 1) * 2^i
+     * for maximum hash value is \sum from (i=0 to MAX_HASH_LENGTH - 1) with (HASH_MODULO - 1) * 2^i
      */
     private static final int MAX_HASH_LENGTH = 25;
-    private static final int TOKEN_HASH_MODULO = 64;
+    private static final int HASH_MODULO = 64;
 
     /** Indicator that the subsequence should not be considered for comparison matching */
     public static final int NO_HASH = -1;
 
     private final int windowSize;
-    private final List<Token> tokens;
+    private final int[] values;
     private int[] subsequenceHashes;
     private Map<Integer, List<Integer>> startIndexToSubsequenceHashesMap;
 
     /**
      * Generates a new subsequence hash lookup table. Performance is optimized to compute hashes in O(n).
      * @param windowSize the size of the subsequences.
-     * @param tokens the tokens to hash over.
-     * @param markedTokens the set of marked tokens. Subsequences containing a marked token obtain the {@link #NO_HASH}
+     * @param values the values to hash over.
+     * @param markedIndexes the indexes of marked values. Subsequences containing a marked value obtain the {@link #NO_HASH}
      * value.
      */
-    SubsequenceHashLookupTable(int windowSize, List<Token> tokens, Set<Token> markedTokens) {
+    SubsequenceHashLookupTable(int windowSize, int[] values, Set<Integer> markedIndexes) {
         windowSize = Math.max(1, windowSize);
         windowSize = Math.min(MAX_HASH_LENGTH, windowSize);
         this.windowSize = windowSize;
-        this.tokens = tokens;
+        this.values = values;
 
-        if (tokens.size() < windowSize) {
+        if (values.length < windowSize) {
             return;
         }
 
-        subsequenceHashes = new int[tokens.size() - windowSize];
+        subsequenceHashes = new int[values.length - windowSize];
         startIndexToSubsequenceHashesMap = new HashMap<>(subsequenceHashes.length);
-        computeSubsequenceHashes(markedTokens);
+        computeSubsequenceHashes(markedIndexes);
     }
 
     /** Returns the size of the subsequences used for hashing */
@@ -53,9 +53,9 @@ class SubsequenceHashLookupTable {
         return windowSize;
     }
 
-    /** Returns the list of tokens for which the hashes were computed */
-    List<Token> getTokens() {
-        return tokens;
+    /** Returns the list of values for which the hashes were computed */
+    int[] getValues() {
+        return values;
     }
 
     /**
@@ -81,16 +81,16 @@ class SubsequenceHashLookupTable {
 
     /**
      * Creates hashes for all subsequences with windowSize. Code is optimized to perform in O(n) using a windowing approach.
-     * Hashes are computed by \sum from (i=0 to windowSize) with hash(tokens[offset+i]) * 2^(hashLength-1-i)
-     * @param markedTokens contains the marked tokens. Subsequences containing a marked token will receive the NO_HASH
-     * value.
+     * Hashes are computed by \sum from (i=0 to windowSize) with hash(values[offset+i]) * 2^(hashLength-1-i)
+     * @param markedIndexes contains the indexes of marked values. Subsequences containing a marked value will receive the
+     * {@link #NO_HASH} value.
      */
-    private void computeSubsequenceHashes(Set<Token> markedTokens) {
+    private void computeSubsequenceHashes(Set<Integer> markedIndexes) {
         int hash = 0;
         int hashedLength = 0;
         int factor = (windowSize != 1 ? (2 << (windowSize - 2)) : 1);
 
-        for (int windowEndIndex = 0; windowEndIndex < tokens.size(); windowEndIndex++) {
+        for (int windowEndIndex = 0; windowEndIndex < values.length; windowEndIndex++) {
             int windowStartIndex = windowEndIndex - windowSize;
             if (windowStartIndex >= 0) {
                 if (hashedLength >= windowSize) {
@@ -99,10 +99,10 @@ class SubsequenceHashLookupTable {
                 } else {
                     subsequenceHashes[windowStartIndex] = NO_HASH;
                 }
-                hash -= factor * (hashValueForToken(tokens.get(windowStartIndex)));
+                hash -= factor * hashValueForValue(values[windowStartIndex]);
             }
-            hash = (2 * hash) + (hashValueForToken(tokens.get(windowEndIndex)));
-            if (markedTokens.contains(tokens.get(windowEndIndex))) {
+            hash = (2 * hash) + hashValueForValue(values[windowEndIndex]);
+            if (markedIndexes.contains(windowEndIndex)) {
                 hashedLength = 0;
             } else {
                 hashedLength++;
@@ -110,8 +110,8 @@ class SubsequenceHashLookupTable {
         }
     }
 
-    private int hashValueForToken(Token token) {
-        return token.type % TOKEN_HASH_MODULO;
+    private int hashValueForValue(int value) {
+        return value % HASH_MODULO;
     }
 
     private void addToStartIndexesToHashesMap(int startIndex, int subsequenceHash) {

language-api/src/main/java/de/jplag/SharedTokenType.java

@@ -0,0 +1,26 @@
+package de.jplag;
+
+/**
+ * Shared token types that occur for any language.
+ */
+public enum SharedTokenType implements TokenType {
+    /**
+     * Marks the end of the file. Every parsed file must have this token type as its last element.
+     */
+    FILE_END("EOF");
+
+    private final String description;
+
+    public String getDescription() {
+        return description;
+    }
+
+    SharedTokenType(String description) {
+        this.description = description;
+    }
+
+    @Override
+    public Boolean isExcludedFromMatching() {
+        return true;
+    }
+}

language-api/src/main/java/de/jplag/Token.java

@@ -1,11 +1,15 @@
 package de.jplag;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * This class represents a token in a source code. It can represent keywords, identifiers, syntactical structures etc.
  * What types of tokens there are depends on the specific language, meaning JPlag does not enforce a specific token set.
  * The language parsers decide what is a token and what is not.
  */
-public abstract class Token {
+public class Token {
+    private Logger logger = LoggerFactory.getLogger(this.getClass());
     /** Indicates that the requested field has no value. */
     public static final int NO_VALUE = -1;
 
@@ -13,36 +17,34 @@ public abstract class Token {
     private int column;
     private int length;
     private String file;
-
-    protected int type;
+    private TokenType type;
 
     /**
-     * Creates a token without information about the column or the length of the token in the line.
-     * @param type is the token type.
+     * Creates a token of type {@link SharedTokenType#FILE_END FILE_END} without information about line, column, and length.
      * @param file is the name of the source code file.
-     * @param line is the line index in the source code where the token resides. Cannot be smaller than 1. For
-     * {@link TokenConstants#FILE_END FILE_END} it is automatically set to {@link #NO_VALUE}.
      */
-    public Token(int type, String file, int line) {
-        this.type = type;
-        this.file = file;
-        if (type == TokenConstants.FILE_END) {
-            this.line = NO_VALUE;
-        } else {
-            this.line = line > 0 ? line : 1;
-        }
+    public static Token fileEnd(String file) {
+        return new Token(SharedTokenType.FILE_END, file, NO_VALUE, NO_VALUE, NO_VALUE);
     }
 
     /**
      * Creates a token with column and length information.
      * @param type is the token type.
      * @param file is the name of the source code file.
-     * @param line is the line index in the source code where the token resides. Cannot be smaller than 1.
-     * @param column is the column index, meaning where the token starts in the line.
+     * @param line is the line index in the source code where the token resides. Index is 1-based.
+     * @param column is the column index, meaning where the token starts in the line. Index is 1-based.
      * @param length is the length of the token in the source code.
      */
-    public Token(int type, String file, int line, int column, int length) {
-        this(type, file, line);
+    public Token(TokenType type, String file, int line, int column, int length) {
+        if (line == 0) {
+            logger.warn("Creating a token with line index 0 while index is 1-based");
+        }
+        if (column == 0) {
+            logger.warn("Creating a token with column index 0 while index is 1-based");
+        }
+        this.type = type;
+        this.file = file;
+        this.line = line;
         this.column = column;
         this.length = length;
     }
@@ -79,27 +81,9 @@ public abstract class Token {
     }
 
     /**
-     * @return the type identifier of the token.
+     * @return the type of the token.
      */
-    public int getType() {
+    public TokenType getType() {
         return type;
     }
-
-    /**
-     * Sets the character index which denotes where the code sections represented by this token starts in the line.
-     * @param column is the index in characters to set.
-     */
-    public void setColumn(int column) {
-        this.column = column;
-    }
-
-    @Override
-    public String toString() {
-        return type2string();
-    }
-
-    /**
-     * @return a string representation depending on the type of the token.
-     */
-    protected abstract String type2string();
 }

language-api/src/main/java/de/jplag/TokenConstants.java

@@ -1,16 +0,0 @@
-package de.jplag;
-
-/**
- * Super interface for all language parsers. Provides the only two token that are required from every parser.
- */
-public interface TokenConstants {
-    /**
-     * Marks the end of the file, has a special purpose in the comparison algorithm.
-     */
-    int FILE_END = 0;
-
-    /**
-     * Used to optionally separate methods from each other with an always marked token.
-     */
-    int SEPARATOR_TOKEN = 1;
-}

language-api/src/main/java/de/jplag/TokenPrinter.java

@@ -132,7 +132,7 @@ public final class TokenPrinter {
             if (token.getLine() != Token.NO_VALUE) {
                 currentLine = token.getLine();
             }
-            int line = token.getType() == TokenConstants.FILE_END ? lines.size() : currentLine;
+            int line = token.getType() == SharedTokenType.FILE_END ? lines.size() : currentLine;
             List<Token> tokens = lineNumbersToTokens.containsKey(line) ? lineNumbersToTokens.get(line) : new ArrayList<>();
             tokens.add(token);
             lineNumbersToTokens.put(line, tokens);
@@ -152,7 +152,8 @@ public final class TokenPrinter {
      * Determines the string representation of the token.
      */
     private static String getStringRepresentation(Token token) {
-        return token.getLength() <= 1 && INDICATE_TINY_TOKEN ? token.toString().toLowerCase() : token.toString();
+        String description = token.getType().getDescription();
+        return token.getLength() <= 1 && INDICATE_TINY_TOKEN ? description.toLowerCase() : description;
     }
 
     /**

language-api/src/main/java/de/jplag/TokenType.java

@@ -0,0 +1,22 @@
+package de.jplag;
+
+/**
+ * Indicates the type of a token. Needs to be implemented for each language module to declare what types of tokens can
+ * be extracted from code written in that language. A token type is expected to be stateless, thus it is recommended to
+ * use an <code>enum</code> or <code>record</code>.
+ * @see SharedTokenType
+ */
+public interface TokenType {
+    /**
+     * Returns the user-readable description of this token type.
+     */
+    String getDescription();
+
+    /**
+     * Indicates that no matches containing this token type shall be generated. Defaults to <code>false</code>.
+     * @return <code>true</code> if token type is excluded from matching, otherwise <code>false</code>.
+     */
+    default Boolean isExcludedFromMatching() {
+        return false;
+    }
+}

language-api/src/test/java/de/jplag/TestToken.java

@@ -1,19 +0,0 @@
-package de.jplag;
-
-import static de.jplag.TokenConstants.FILE_END;
-import static de.jplag.simple.TestTokenConstants.STRING;
-
-public class TestToken extends Token {
-    public TestToken(int tokenType, String file, int line, int column, int length) {
-        super(tokenType, file, line, column, length);
-    }
-
-    @Override
-    protected String type2string() {
-        return switch (type) {
-            case STRING -> "STRING";
-            case FILE_END -> "<EOF>";
-            default -> "<UNKNOWN%d>".formatted(type);
-        };
-    }
-}

language-api/src/test/java/de/jplag/TokenPrinterTest.java

@@ -1,7 +1,6 @@
 package de.jplag;
 
-import static de.jplag.TokenConstants.FILE_END;
-import static de.jplag.simple.TestTokenConstants.STRING;
+import static de.jplag.SharedTokenType.FILE_END;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -29,37 +28,37 @@ class TokenPrinterTest {
 
         // See TokenPrinterTest.txt for the intended behaviour
         List<Token> tokens = new ArrayList<>();
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 1, 1, "STRING".length()));
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 2, 1, "STRING".length() + 1));
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 3, 1, "STRING".length() + 2));
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 4, 1, "STRING".length() + 10));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 1, 1, "STRING".length()));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 2, 1, "STRING".length() + 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 3, 1, "STRING".length() + 2));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 4, 1, "STRING".length() + 10));
 
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 6, 3, 1));
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 7, 9, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 6, 3, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 7, 9, 1));
 
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 9, 1, 1));
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 9, 10, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 9, 1, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 9, 10, 1));
 
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 10, 1, 1));
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 10, 5, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 10, 1, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 10, 5, 1));
 
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 12, 1, 1));
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 12, 5, 1));
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 12, 10, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 12, 1, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 12, 5, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 12, 10, 1));
 
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 14, 10, 1));
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 14, 5, 1));
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 14, 1, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 14, 10, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 14, 5, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 14, 1, 1));
 
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 16, -5, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 16, -5, 1));
 
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 19, 100, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 19, 100, 1));
 
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 22, 1, 100));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 22, 1, 100));
 
-        tokens.add(new TestToken(FILE_END, TEST_FILE_NAME, Token.NO_VALUE, Token.NO_VALUE, Token.NO_VALUE));
+        tokens.add(new Token(FILE_END, TEST_FILE_NAME, Token.NO_VALUE, Token.NO_VALUE, Token.NO_VALUE));
 
-        tokens.add(new TestToken(STRING, TEST_FILE_NAME, 100, 1, 1));
+        tokens.add(new Token(TestTokenType.STRING, TEST_FILE_NAME, 100, 1, 1));
 
         String output = TokenPrinter.printTokens(tokens, TEST_FILE_LOCATION.toFile());
         logger.info(output); // no additional newline required
@@ -86,7 +85,8 @@ class TokenPrinterTest {
                         continue;
                     }
                     Token currentToken = tokens.get(tokenIndex);
-                    assertTrue(lineToken.equalsIgnoreCase(currentToken.toString()), "expected: %s, actual: %s".formatted(lineToken, currentToken));
+                    assertTrue(lineToken.equalsIgnoreCase(currentToken.getType().getDescription()),
+                            "expected: %s, actual: %s".formatted(lineToken, currentToken));
                     if (currentToken.getLine() != Token.NO_VALUE) {
                         assertEquals(lineIndex, currentToken.getLine(), "invalid line for token " + currentToken);
                     }
@@ -96,4 +96,18 @@ class TokenPrinterTest {
         }
         assertEquals(tokens.size() - 1, tokenIndex, "incorrect number of tokens printed");
     }
+
+    private enum TestTokenType implements TokenType {
+        STRING("STRING");
+
+        private final String description;
+
+        public String getDescription() {
+            return description;
+        }
+
+        TestTokenType(String description) {
+            this.description = description;
+        }
+    }
 }

language-testutils/src/test/java/de/jplag/testutils/TokenUtils.java

@@ -3,6 +3,7 @@ package de.jplag.testutils;
 import java.util.List;
 
 import de.jplag.Token;
+import de.jplag.TokenType;
 
 public final class TokenUtils {
 
@@ -12,17 +13,17 @@ public final class TokenUtils {
 
     /**
      * Returns the type of all tokens that belong to a certain file.
-     * @param tokenList is the list of {@link Token Tokens}.
+     * @param tokens is the list of {@link Token Tokens}.
      * @param name is the name of the target file.
      * @return the immutable list of token types.
      */
-    public static List<Integer> tokenTypesByFile(List<Token> tokens, String name) {
+    public static List<TokenType> tokenTypesByFile(List<Token> tokens, String name) {
         return tokensByFile(tokens, name).stream().map(Token::getType).toList();
     }
 
     /**
      * Returns the tokens that belong to a certain file.
-     * @param tokenList is the list of {@link Token Tokens}.
+     * @param tokens is the list of {@link Token Tokens}.
      * @param name is the name of the target file.
      * @return the immutable list of tokens.
      */

languages/cpp/src/main/java/de/jplag/cpp/CPPToken.java

@@ -1,82 +0,0 @@
-package de.jplag.cpp;
-
-import de.jplag.Token;
-
-public class CPPToken extends Token implements CPPTokenConstants {
-
-    public CPPToken(int type, String file, int line, int column, int length) {
-        super(type, file, line, column, length);
-    }
-
-    public CPPToken(int type, String file) {
-        super(type, file, 1, 0, 0);
-    }
-
-    protected String type2string() {
-        return switch (type) {
-            case CPPTokenConstants.FILE_END -> "EOF";
-            case C_BLOCK_BEGIN -> "BLOCK{";
-            case C_BLOCK_END -> "}BLOCK";
-            case C_SCOPE -> "SCOPE";
-            case C_QUESTIONMARK -> "COND";
-            case C_ELLIPSIS -> "...";
-            case C_ASSIGN -> "ASSIGN";
-            case C_DOT -> "DOT";
-            case C_ARROW -> "ARROW";
-            case C_DOTSTAR -> "DOTSTAR";
-            case C_ARROWSTAR -> "ARROWSTAR";
-            case C_AUTO -> "AUTO";
-            case C_BREAK -> "BREAK";
-            case C_CASE -> "CASE";
-            case C_CATCH -> "CATCH";
-            case C_CHAR -> "CHAR";
-            case C_CONST -> "CONST";
-            case C_CONTINUE -> "CONTINUE";
-            case C_DEFAULT -> "DEFAULT";
-            case C_DELETE -> "DELETE";
-            case C_DO -> "DO";
-            case C_DOUBLE -> "DOUBLE";
-            case C_ELSE -> "ELSE";
-            case C_ENUM -> "ENUM";
-            case C_EXTERN -> "EXTERN";
-            case C_FLOAT -> "FLOAT";
-            case C_FOR -> "FOR";
-            case C_FRIEND -> "FRIEND";
-            case C_GOTO -> "GOTO";
-            case C_IF -> "IF";
-            case C_INLINE -> "INLINE";
-            case C_INT -> "INT";
-            case C_LONG -> "LONG";
-            case C_NEW -> "NEW";
-            case C_PRIVATE -> "PRIVATE";
-            case C_PROTECTED -> "PROTECTED";
-            case C_PUBLIC -> "PUBLIC";
-            case C_REDECLARED -> "REDECLARED";
-            case C_REGISTER -> "REGISTER";
-            case C_RETURN -> "RETURN";
-            case C_SHORT -> "SHORT";
-            case C_SIGNED -> "SIGNED";
-            case C_SIZEOF -> "SIZEOF";
-            case C_STATIC -> "STATIC";
-            case C_STRUCT -> "STRUCT";
-            case C_CLASS -> "CLASS";
-            case C_SWITCH -> "SWITCH";
-            case C_TEMPLATE -> "TEMPLATE";
-            case C_THIS -> "THIS";
-            case C_TRY -> "TRY";
-            case C_TYPEDEF -> "TYPEDEF";
-            case C_UNION -> "UNION";
-            case C_UNSIGNED -> "UNSIGNED";
-            case C_VIRTUAL -> "VIRTUAL";
-            case C_VOID -> "VOID";
-            case C_VOLANTILE -> "VOLANTILE";
-            case C_WHILE -> "WHILE";
-            case C_OPERATOR -> "OPERATOR";
-            case C_THROW -> "THROW";
-            case C_ID -> "ID";
-            case C_FUN -> "FUN";
-            case C_NULL -> "NULL";
-            default -> "<UNKNOWN" + type + ">";
-        };
-    }
-}

languages/cpp/src/main/java/de/jplag/cpp/CPPTokenConstants.java

@@ -1,69 +0,0 @@
-package de.jplag.cpp;
-
-import de.jplag.TokenConstants;
-
-public interface CPPTokenConstants extends TokenConstants {
-    int C_BLOCK_BEGIN = 2;
-    int C_BLOCK_END = 3;
-    int C_SCOPE = 4;
-    int C_QUESTIONMARK = 5;
-    int C_ELLIPSIS = 6;
-    int C_ASSIGN = 7;
-    int C_DOT = 8;
-    int C_ARROW = 9;
-    int C_ARROWSTAR = 10;
-    int C_AUTO = 11;
-    int C_BREAK = 12;
-    int C_CASE = 13;
-    int C_CATCH = 14;
-    int C_CHAR = 15;
-    int C_CONST = 16;
-    int C_CONTINUE = 17;
-    int C_DEFAULT = 18;
-    int C_DELETE = 19;
-    int C_DO = 20;
-    int C_DOUBLE = 21;
-    int C_ELSE = 22;
-    int C_ENUM = 23;
-    int C_EXTERN = 24;
-    int C_FLOAT = 25;
-    int C_FOR = 26;
-    int C_FRIEND = 27;
-    int C_GOTO = 28;
-    int C_IF = 29;
-    int C_INLINE = 30;
-    int C_INT = 31;
-    int C_LONG = 32;
-    int C_NEW = 33;
-    int C_PRIVATE = 34;
-    int C_PROTECTED = 35;
-    int C_PUBLIC = 36;
-    int C_REDECLARED = 37;
-    int C_REGISTER = 38;
-    int C_RETURN = 39;
-    int C_SHORT = 40;
-    int C_SIGNED = 41;
-    int C_SIZEOF = 42;
-    int C_STATIC = 43;
-    int C_STRUCT = 44;
-    int C_CLASS = 45;
-    int C_SWITCH = 46;
-    int C_TEMPLATE = 47;
-    int C_THIS = 48;
-    int C_TRY = 49;
-    int C_TYPEDEF = 50;
-    int C_UNION = 51;
-    int C_UNSIGNED = 52;
-    int C_VIRTUAL = 53;
-    int C_VOID = 54;
-    int C_VOLANTILE = 55;
-    int C_WHILE = 56;
-    int C_OPERATOR = 57;
-    int C_THROW = 58;
-    int C_ID = 59;
-    int C_FUN = 60;
-    int C_DOTSTAR = 61;
-    int C_NULL = 62;
-
-    int NUM_DIFF_TOKENS = 63;
-}

languages/cpp/src/main/java/de/jplag/cpp/CPPTokenType.java

@@ -0,0 +1,77 @@
+package de.jplag.cpp;
+
+import de.jplag.TokenType;
+
+public enum CPPTokenType implements TokenType {
+    C_BLOCK_BEGIN("BLOCK{"),
+    C_BLOCK_END("}BLOCK"),
+    C_SCOPE("SCOPE"),
+    C_QUESTIONMARK("COND"),
+    C_ELLIPSIS("..."),
+    C_ASSIGN("ASSIGN"),
+    C_DOT("DOT"),
+    C_ARROW("ARROW"),
+    C_ARROWSTAR("ARROWSTAR"),
+    C_AUTO("AUTO"),
+    C_BREAK("BREAK"),
+    C_CASE("CASE"),
+    C_CATCH("CATCH"),
+    C_CHAR("CHAR"),
+    C_CONST("CONST"),
+    C_CONTINUE("CONTINUE"),
+    C_DEFAULT("DEFAULT"),
+    C_DELETE("DELETE"),
+    C_DO("DO"),
+    C_DOUBLE("DOUBLE"),
+    C_ELSE("ELSE"),
+    C_ENUM("ENUM"),
+    C_EXTERN("EXTERN"),
+    C_FLOAT("FLOAT"),
+    C_FOR("FOR"),
+    C_FRIEND("FRIEND"),
+    C_GOTO("GOTO"),
+    C_IF("IF"),
+    C_INLINE("INLINE"),
+    C_INT("INT"),
+    C_LONG("LONG"),
+    C_NEW("NEW"),
+    C_PRIVATE("PRIVATE"),
+    C_PROTECTED("PROTECTED"),
+    C_PUBLIC("PUBLIC"),
+    C_REDECLARED("REDECLARED"),
+    C_REGISTER("REGISTER"),
+    C_RETURN("RETURN"),
+    C_SHORT("SHORT"),
+    C_SIGNED("SIGNED"),
+    C_SIZEOF("SIZEOF"),
+    C_STATIC("STATIC"),
+    C_STRUCT("STRUCT"),
+    C_CLASS("CLASS"),
+    C_SWITCH("SWITCH"),
+    C_TEMPLATE("TEMPLATE"),
+    C_THIS("THIS"),
+    C_TRY("TRY"),
+    C_TYPEDEF("TYPEDEF"),
+    C_UNION("UNION"),
+    C_UNSIGNED("UNSIGNED"),
+    C_VIRTUAL("VIRTUAL"),
+    C_VOID("VOID"),
+    C_VOLATILE("VOLATILE"),
+    C_WHILE("WHILE"),
+    C_OPERATOR("OPERATOR"),
+    C_THROW("THROW"),
+    C_ID("ID"),
+    C_FUN("FUN"),
+    C_DOTSTAR("DOTSTAR"),
+    C_NULL("NULL");
+
+    private final String description;
+
+    public String getDescription() {
+        return this.description;
+    }
+
+    CPPTokenType(String description) {
+        this.description = description;
+    }
+}

languages/cpp/src/main/java/de/jplag/cpp/Scanner.java

@@ -5,12 +5,12 @@ import java.util.ArrayList;
 import java.util.List;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenConstants;
+import de.jplag.Token;
 
 public class Scanner extends AbstractParser {
     private String currentFile;
 
-    private List<de.jplag.Token> tokens;
+    private List<Token> tokens;
 
     /**
      * Creates the parser.
@@ -19,7 +19,7 @@ public class Scanner extends AbstractParser {
         super();
     }
 
-    public List<de.jplag.Token> scan(File directory, String[] files) {
+    public List<Token> scan(File directory, String[] files) {
         tokens = new ArrayList<>();
         errors = 0;
         for (String currentFile : files) {
@@ -28,13 +28,13 @@ public class Scanner extends AbstractParser {
             if (!CPPScanner.scanFile(directory, currentFile, this)) {
                 errors++;
             }
-            tokens.add(new CPPToken(TokenConstants.FILE_END, currentFile));
+            tokens.add(Token.fileEnd(currentFile));
         }
         return tokens;
     }
 
-    public void add(int type, Token token) {
+    public void add(CPPTokenType type, de.jplag.cpp.Token token) {
         int length = token.endColumn - token.beginColumn + 1;
-        tokens.add(new CPPToken(type, currentFile, token.beginLine, token.beginColumn, length));
+        tokens.add(new Token(type, currentFile, token.beginLine, token.beginColumn, length));
     }
 }

languages/cpp/src/main/javacc/CPP.jj

@@ -19,7 +19,9 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 
-public class CPPScanner implements CPPTokenConstants {
+import static de.jplag.cpp.CPPTokenType.*;
+
+public class CPPScanner {
     private Scanner delegatingScanner;
 
     public static boolean scanFile(File dir, String fileName, Scanner delegatingScanner) {
@@ -362,7 +364,7 @@ void token(): {}
 | "unsigned"     { delegatingScanner.add(C_UNSIGNED,token); }
 | "virtual"      { delegatingScanner.add(C_VIRTUAL,token); }
 | "void"         { delegatingScanner.add(C_VOID,token); }
-| "volatile"     { delegatingScanner.add(C_VOLANTILE,token); }
+| "volatile"     { delegatingScanner.add(C_VOLATILE,token); }
 | "while"        { delegatingScanner.add(C_WHILE,token); }
 | "operator"     { delegatingScanner.add(C_OPERATOR,token); }
 | "true"

languages/csharp/src/main/java/de/jplag/csharp/CSharpListener.java

@@ -1,5 +1,7 @@
 package de.jplag.csharp;
 
+import static de.jplag.csharp.CSharpTokenType.*;
+
 import org.antlr.v4.runtime.Token;
 
 import de.jplag.csharp.grammar.CSharpParser.Accessor_bodyContext;
@@ -60,7 +62,7 @@ import de.jplag.csharp.grammar.CSharpParserBaseListener;
  * Listener class for visiting the C# ANTLR parse tree. Transforms selected ANTLR token into JPlag tokens.
  * @author Timur Saglam
  */
-public class CSharpListener extends CSharpParserBaseListener implements CSharpTokenConstants {
+public class CSharpListener extends CSharpParserBaseListener {
 
     private final CSharpParserAdapter parserAdapter;
 
@@ -77,7 +79,7 @@ public class CSharpListener extends CSharpParserBaseListener implements CSharpTo
      * @param targetType is the type of the JPlag token to be created.
      * @param token is the ANTLR token.
      */
-    private void transformToken(int targetType, Token token) {
+    private void transformToken(CSharpTokenType targetType, Token token) {
         parserAdapter.addToken(targetType, token.getLine(), token.getCharPositionInLine() + 1, token.getText().length());
     }
 

languages/csharp/src/main/java/de/jplag/csharp/CSharpParserAdapter.java

@@ -14,7 +14,7 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
 import de.jplag.Token;
-import de.jplag.TokenConstants;
+import de.jplag.TokenType;
 import de.jplag.csharp.grammar.CSharpLexer;
 import de.jplag.csharp.grammar.CSharpParser;
 
@@ -47,7 +47,7 @@ public class CSharpParserAdapter extends AbstractParser {
             if (!parseFile(directory, fileName)) {
                 errors++;
             }
-            tokens.add(new CSharpToken(TokenConstants.FILE_END, fileName, -1, -1, -1));
+            tokens.add(Token.fileEnd(fileName));
         }
         return tokens;
     }
@@ -78,7 +78,7 @@ public class CSharpParserAdapter extends AbstractParser {
         return true;
     }
 
-    /* package-private */ void addToken(int type, int line, int column, int length) {
-        tokens.add(new CSharpToken(type, currentFile, line, column, length));
+    /* package-private */ void addToken(TokenType type, int line, int column, int length) {
+        tokens.add(new Token(type, currentFile, line, column, length));
     }
 }

languages/csharp/src/main/java/de/jplag/csharp/CSharpToken.java

@@ -1,93 +0,0 @@
-package de.jplag.csharp;
-
-import de.jplag.Token;
-import de.jplag.TokenConstants;
-
-/**
- * C# token class.
- * @author Timur Saglam
- */
-public class CSharpToken extends Token implements CSharpTokenConstants {
-
-    /**
-     * Creates a C# token.
-     * @param type is the corresponding ID of the {@link TokenConstants}.
-     * @param file is the name of the source code file.
-     * @param line is the line index in the source code where the token resides. Cannot be smaller than 1.
-     * @param column is the column index, meaning where the token starts in the line.
-     * @param length is the length of the token in the source code.
-     */
-    public CSharpToken(int type, String file, int line, int column, int length) {
-        super(type, file, line, column, length);
-    }
-
-    @Override
-    protected String type2string() {
-        return switch (type) {
-            case TokenConstants.FILE_END -> "EOF";
-            case INVOCATION -> "INVOC";
-            case OBJECT_CREATION -> "OBJECT";
-            case ARRAY_CREATION -> "ARRAY";
-            case ASSIGNMENT -> "ASSIGN";
-            case FIELD -> "FIELD";
-            case CONSTANT -> "CONST";
-            case IF -> "IF";
-            case IF_END -> "}IF";
-            case SWITCH_BEGIN -> "SWITCH{";
-            case SWITCH_END -> "}SWITCH";
-            case CASE -> "CASE";
-            case DO -> "DO";
-            case WHILE -> "WHILE";
-            case FOR -> "FOR";
-            case FOREACH -> "FOREACH";
-            case BREAK -> "BREAK";
-            case CONTINUE -> "CONTINUE";
-            case GOTO -> "GOTO";
-            case RETURN -> "RETURN";
-            case THROW -> "THROW";
-            case CHECKED -> "CHECKED";
-            case UNCHECKED -> "UNCHECKED";
-            case LOCK -> "LOCK";
-            case USING -> "USING";
-            case TRY -> "TRY";
-            case CATCH -> "CATCH";
-            case FINALLY -> "FINALLY";
-            case NAMESPACE_BEGIN -> "NAMESPACE{";
-            case NAMESPACE_END -> "}NAMESPACE";
-            case USING_DIRECTIVE -> "USING";
-            case CLASS_BEGIN -> "CLASS{";
-            case CLASS_END -> "}CLASS";
-            case METHOD -> "METHOD";
-            case PROPERTY -> "PROPERTY";
-            case EVENT -> "EVENT";
-            case INDEXER -> "INDEXER";
-            case OPERATOR -> "OPERATOR";
-            case CONSTRUCTOR -> "CONSTR";
-            case DESTRUCTOR -> "DESTRUCTOR";
-            case STRUCT_BEGIN -> "STRUCT{";
-            case STRUCT_END -> "}STRUCT";
-            case INTERFACE_BEGIN -> "INTERFACE{";
-            case INTERFACE_END -> "}INTERFACE";
-            case ENUM -> "ENUM";
-            case DELEGATE -> "DELEGATE";
-            case ATTRIBUTE -> "ATTRIBUTE";
-            case UNSAFE -> "UNSAFE";
-            case FIXED -> "FIXED";
-            case METHOD_BEGIN -> "METHOD{";
-            case METHOD_END -> "}METHOD";
-            case STRUCT -> "STRUCT";
-            case IF_BEGIN -> "IF{";
-            case CLASS -> "CLASS";
-            case INTERFACE -> "INTERFACE";
-            case ENUM_BEGIN -> "ENUM{";
-            case ENUM_END -> "}ENUM";
-            case ENUMERAL -> "ENUMERAL";
-            case LOCAL_VARIABLE -> "LOCAL_VAR";
-            case ACCESSORS_BEGIN -> "ACCESSORS{";
-            case ACCESSORS_END -> "}ACCESSORS";
-            case ACCESSOR_BEGIN -> "ACCESSORS{";
-            case ACCESSOR_END -> "}ACCESSORS";
-            default -> "<UNKNOWN" + type + ">";
-        };
-    }
-}

languages/csharp/src/main/java/de/jplag/csharp/CSharpTokenConstants.java

@@ -1,80 +0,0 @@
-package de.jplag.csharp;
-
-import de.jplag.TokenConstants;
-
-/**
- * Based on the legacy CSharp-1.2 constants.
- * @author Timur Saglam
- */
-public interface CSharpTokenConstants extends TokenConstants {
-    /*
-     * Token constants:
-     */
-    int INVOCATION = 2;
-    int OBJECT_CREATION = 3;
-    int ARRAY_CREATION = 4;
-    int ASSIGNMENT = 5;
-    int FIELD = 6;
-    int CONSTANT = 7;
-    int LOCAL_VARIABLE = 8;
-    int IF = 9;
-    int SWITCH_BEGIN = 10;
-    int SWITCH_END = 11;
-    int CASE = 12;
-    int DO = 13;
-    int WHILE = 14;
-    int FOR = 15;
-    int FOREACH = 16;
-    int BREAK = 17;
-    int CONTINUE = 18;
-    int GOTO = 19;
-    int RETURN = 20;
-    int THROW = 21;
-    int CHECKED = 22;
-    int UNCHECKED = 23;
-    int LOCK = 24;
-    int USING = 25;
-    int TRY = 26;
-    int CATCH = 27;
-    int FINALLY = 28;
-    int NAMESPACE_BEGIN = 29;
-    int NAMESPACE_END = 30;
-    int USING_DIRECTIVE = 31;
-    int CLASS_BEGIN = 32;
-    int CLASS_END = 33;
-    int METHOD = 34;
-    int PROPERTY = 35;
-    int EVENT = 36;
-    int INDEXER = 37;
-    int OPERATOR = 38;
-    int CONSTRUCTOR = 39;
-    int DESTRUCTOR = 40;
-    int STRUCT_BEGIN = 41;
-    int STRUCT_END = 42;
-    int INTERFACE_BEGIN = 43;
-    int INTERFACE_END = 44;
-    int ENUM = 45;
-    int DELEGATE = 46;
-    int ATTRIBUTE = 47;
-    int IF_END = 48;
-    int UNSAFE = 49;
-    int FIXED = 50;
-    int METHOD_BEGIN = 51;
-    int METHOD_END = 52;
-    int STRUCT = 53;
-    int IF_BEGIN = 54;
-    int CLASS = 55;
-    int INTERFACE = 56;
-    int ENUM_BEGIN = 57;
-    int ENUM_END = 58;
-    int ENUMERAL = 59;
-    int ACCESSORS_BEGIN = 60;
-    int ACCESSORS_END = 61;
-    int ACCESSOR_BEGIN = 62;
-    int ACCESSOR_END = 63;
-
-    /*
-     * Number of token constants:
-     */
-    int NUM_DIFF_TOKENS = 64;
-}

languages/csharp/src/main/java/de/jplag/csharp/CSharpTokenType.java

@@ -0,0 +1,81 @@
+package de.jplag.csharp;
+
+import de.jplag.TokenType;
+
+/**
+ * Based on the legacy C# 1.2 constants.
+ */
+public enum CSharpTokenType implements TokenType {
+    INVOCATION("INVOC"),
+    OBJECT_CREATION("OBJECT"),
+    ARRAY_CREATION("ARRAY"),
+    ASSIGNMENT("ASSIGN"),
+    FIELD("FIELD"),
+    CONSTANT("CONST"),
+    LOCAL_VARIABLE("LOCAL_VAR"),
+    IF("IF"),
+    SWITCH_BEGIN("SWITCH{"),
+    SWITCH_END("}SWITCH"),
+    CASE("CASE"),
+    DO("DO"),
+    WHILE("WHILE"),
+    FOR("FOR"),
+    FOREACH("FOREACH"),
+    BREAK("BREAK"),
+    CONTINUE("CONTINUE"),
+    GOTO("GOTO"),
+    RETURN("RETURN"),
+    THROW("THROW"),
+    CHECKED("CHECKED"),
+    UNCHECKED("UNCHECKED"),
+    LOCK("LOCK"),
+    USING("USING"),
+    TRY("TRY"),
+    CATCH("CATCH"),
+    FINALLY("FINALLY"),
+    NAMESPACE_BEGIN("NAMESPACE{"),
+    NAMESPACE_END("}NAMESPACE"),
+    USING_DIRECTIVE("USING"),
+    CLASS_BEGIN("CLASS{"),
+    CLASS_END("}CLASS"),
+    METHOD("METHOD"),
+    PROPERTY("PROPERTY"),
+    EVENT("EVENT"),
+    INDEXER("INDEXER"),
+    OPERATOR("OPERATOR"),
+    CONSTRUCTOR("CONSTR"),
+    DESTRUCTOR("DESTRUCTOR"),
+    STRUCT_BEGIN("STRUCT{"),
+    STRUCT_END("}STRUCT"),
+    INTERFACE_BEGIN("INTERFACE{"),
+    INTERFACE_END("}INTERFACE"),
+    ENUM("ENUM"),
+    DELEGATE("DELEGATE"),
+    ATTRIBUTE("ATTRIBUTE"),
+    IF_END("}IF"),
+    UNSAFE("UNSAFE"),
+    FIXED("FIXED"),
+    METHOD_BEGIN("METHOD{"),
+    METHOD_END("}METHOD"),
+    STRUCT("STRUCT"),
+    IF_BEGIN("IF{"),
+    CLASS("CLASS"),
+    INTERFACE("INTERFACE"),
+    ENUM_BEGIN("ENUM{"),
+    ENUM_END("}ENUM"),
+    ENUMERAL("ENUMERAL"),
+    ACCESSORS_BEGIN("ACCESSORS{"),
+    ACCESSORS_END("}ACCESSORS"),
+    ACCESSOR_BEGIN("ACCESSOR{"),
+    ACCESSOR_END("}ACCESSOR");
+
+    private final String description;
+
+    public String getDescription() {
+        return this.description;
+    }
+
+    CSharpTokenType(String description) {
+        this.description = description;
+    }
+}

languages/csharp/src/test/java/de/jplag/csharp/MinimalCSharpFrontendTest.java

@@ -1,7 +1,6 @@
 package de.jplag.csharp;
 
-import static de.jplag.csharp.CSharpTokenConstants.*;
-import static java.util.stream.Collectors.toList;
+import static de.jplag.csharp.CSharpTokenType.*;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -14,9 +13,10 @@ import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import de.jplag.SharedTokenType;
 import de.jplag.Token;
-import de.jplag.TokenConstants;
 import de.jplag.TokenPrinter;
+import de.jplag.TokenType;
 
 class MinimalCSharpFrontendTest {
     private final Logger logger = LoggerFactory.getLogger("JPlag-Test");
@@ -36,10 +36,10 @@ class MinimalCSharpFrontendTest {
 
     @Test
     void testParsingTestClass() {
-        List<Integer> expectedToken = List.of(CLASS, CLASS_BEGIN, FIELD, CONSTRUCTOR, LOCAL_VARIABLE, METHOD, METHOD_BEGIN, IF, IF_BEGIN, INVOCATION,
-                IF_END, IF_BEGIN, INVOCATION, IF_END, METHOD_END, PROPERTY, ACCESSORS_BEGIN, ACCESSOR_BEGIN, ACCESSOR_END, ACCESSOR_BEGIN,
+        List<TokenType> expectedToken = List.of(CLASS, CLASS_BEGIN, FIELD, CONSTRUCTOR, LOCAL_VARIABLE, METHOD, METHOD_BEGIN, IF, IF_BEGIN,
+                INVOCATION, IF_END, IF_BEGIN, INVOCATION, IF_END, METHOD_END, PROPERTY, ACCESSORS_BEGIN, ACCESSOR_BEGIN, ACCESSOR_END, ACCESSOR_BEGIN,
                 ACCESSOR_END, ACCESSORS_END, FIELD, PROPERTY, ACCESSORS_BEGIN, ACCESSOR_BEGIN, RETURN, ACCESSOR_END, ACCESSOR_BEGIN, ASSIGNMENT,
-                ACCESSOR_END, ACCESSORS_END, CLASS_END, TokenConstants.FILE_END);
+                ACCESSOR_END, ACCESSORS_END, CLASS_END, SharedTokenType.FILE_END);
 
         // Parse test input
         String[] input = new String[] {TEST_SUBJECT};
@@ -48,7 +48,7 @@ class MinimalCSharpFrontendTest {
 
         // Compare parsed tokens:
         assertEquals(expectedToken.size(), result.size());
-        List<Integer> actualToken = result.stream().map(Token::getType).collect(toList());
+        List<TokenType> actualToken = result.stream().map(Token::getType).toList();
         assertEquals(expectedToken, actualToken);
     }
 

languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/DynamicMetamodelToken.java

@@ -1,7 +1,10 @@
 package de.jplag.emf.dynamic;
 
+import java.util.Optional;
+
 import org.eclipse.emf.ecore.EObject;
 
+import de.jplag.TokenType;
 import de.jplag.emf.MetamodelToken;
 
 /**
@@ -10,20 +13,11 @@ import de.jplag.emf.MetamodelToken;
  */
 public class DynamicMetamodelToken extends MetamodelToken {
 
-    public DynamicMetamodelToken(int type, String file, EObject eObject) {
-        super(type, file, eObject);
+    public DynamicMetamodelToken(TokenType type, String file, EObject eObject) {
+        super(type, file, NO_VALUE, NO_VALUE, NO_VALUE, Optional.of(eObject));
     }
 
-    public DynamicMetamodelToken(int type, String file) {
+    public DynamicMetamodelToken(TokenType type, String file) {
         super(type, file);
     }
-
-    @Override
-    protected String type2string() {
-        if (type < DynamicMetamodelTokenConstants.TOKEN_TYPE_START) {
-            return super.type2string();
-        }
-        return DynamicMetamodelTokenConstants.getTokenString(type);
-    }
-
 }

languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/DynamicMetamodelTokenConstants.java

@@ -1,60 +0,0 @@
-package de.jplag.emf.dynamic;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.eclipse.emf.ecore.EClass;
-
-import de.jplag.TokenConstants;
-
-/**
- * Utility class for the dynamic creation of token constants. Replaces a handcrafted token set.
- * @author Timur Saglam
- */
-public class DynamicMetamodelTokenConstants implements TokenConstants {
-    private static final String UNKNOWN_TOKEN = "<Unknown>";
-
-    public static final int TOKEN_TYPE_START = 2;
-
-    private static Map<EClass, Integer> eClassToTokenType = new HashMap<>();
-    private static Map<Integer, EClass> tokenTypeToEClass = new HashMap<>();
-    private static int tokenTypeIndex = TOKEN_TYPE_START;
-
-    private DynamicMetamodelTokenConstants() {
-        // private constructor for non-instantiability.
-    }
-
-    public static int getTokenType(EClass eClass) {
-        if (eClassToTokenType.containsKey(eClass)) {
-            return eClassToTokenType.get(eClass);
-        }
-        Integer tokenType = tokenTypeIndex++;
-        eClassToTokenType.put(eClass, tokenType);
-        tokenTypeToEClass.put(tokenType, eClass);
-        return tokenType;
-    }
-
-    public static String getTokenString(int tokenType) {
-        if (tokenTypeToEClass.containsKey(tokenType)) {
-            return tokenTypeToEClass.get(tokenType).getName();
-        }
-        return UNKNOWN_TOKEN;
-    }
-
-    /**
-     * Returns the current size of the dynamic token set.
-     * @return the number of known tokens.
-     */
-    public static int getNumberOfTokens() {
-        return eClassToTokenType.size();
-    }
-
-    /**
-     * @return a read only view on the collection of token strings.
-     */
-    public static Collection<String> getTokenStrings() {
-        return eClassToTokenType.keySet().stream().map(EClass::getName).toList();
-    }
-
-}

languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/DynamicMetamodelTokenType.java

@@ -0,0 +1,16 @@
+package de.jplag.emf.dynamic;
+
+import org.eclipse.emf.ecore.EClass;
+import org.eclipse.emf.ecore.EObject;
+
+import de.jplag.TokenType;
+
+public record DynamicMetamodelTokenType(EClass eClass) implements TokenType {
+    public DynamicMetamodelTokenType(EObject eObject) {
+        this(eObject.eClass());
+    }
+
+    public String getDescription() {
+        return eClass.getName();
+    }
+}

languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/parser/DynamicEcoreParser.java

@@ -4,6 +4,7 @@ import org.eclipse.emf.ecore.EObject;
 
 import de.jplag.emf.MetamodelToken;
 import de.jplag.emf.dynamic.DynamicMetamodelToken;
+import de.jplag.emf.dynamic.DynamicMetamodelTokenType;
 import de.jplag.emf.parser.EcoreParser;
 import de.jplag.emf.util.AbstractMetamodelVisitor;
 
@@ -20,8 +21,7 @@ public class DynamicEcoreParser extends EcoreParser {
         return new DynamicMetamodelTokenGenerator(this);
     }
 
-    @Override
-    public void addToken(int type, EObject source) {
+    public void addToken(DynamicMetamodelTokenType type, EObject source) {
         MetamodelToken token = new DynamicMetamodelToken(type, currentFile, source);
         MetamodelToken metadataEnrichedToken = treeView.convertToMetadataEnrichedTokenAndAdd(token, visitor.getCurrentTreeDepth(), NO_PREFIX);
         tokens.add(metadataEnrichedToken);

languages/emf-metamodel-dynamic/src/main/java/de/jplag/emf/dynamic/parser/DynamicMetamodelTokenGenerator.java

@@ -2,7 +2,7 @@ package de.jplag.emf.dynamic.parser;
 
 import org.eclipse.emf.ecore.EObject;
 
-import de.jplag.emf.dynamic.DynamicMetamodelTokenConstants;
+import de.jplag.emf.dynamic.DynamicMetamodelTokenType;
 import de.jplag.emf.util.AbstractMetamodelVisitor;
 
 /**
@@ -23,7 +23,7 @@ public class DynamicMetamodelTokenGenerator extends AbstractMetamodelVisitor {
 
     @Override
     protected void visitEObject(EObject eObject) {
-        int tokenType = DynamicMetamodelTokenConstants.getTokenType(eObject.eClass());
+        var tokenType = new DynamicMetamodelTokenType(eObject);
         parser.addToken(tokenType, eObject);
     }
 }

languages/emf-metamodel-dynamic/src/test/java/de/jplag/emf/dynamic/MinimalDynamicMetamodelTest.java

@@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.File;
 import java.nio.file.Path;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 
@@ -17,6 +18,7 @@ import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
 import de.jplag.TokenPrinter;
+import de.jplag.TokenType;
 import de.jplag.testutils.FileUtil;
 import de.jplag.testutils.TokenUtils;
 
@@ -39,11 +41,11 @@ class MinimalDynamicMetamodelTest {
     @Test
     void testBookstoreMetamodels() {
         List<Token> result = frontend.parse(baseDirectory, TEST_SUBJECTS);
+        List<TokenType> tokenTypes = result.stream().map(Token::getType).toList();
         logger.debug(TokenPrinter.printTokens(result, baseDirectory, Optional.of(Language.VIEW_FILE_SUFFIX)));
-        logger.info(("Dynamic token set: " + DynamicMetamodelTokenConstants.getTokenStrings()));
-        logger.info("parsed tokens: " + result.toString());
-        assertEquals(7, DynamicMetamodelTokenConstants.getTokenStrings().size());
-        assertEquals(64, result.size());
+        logger.info("parsed token types: " + tokenTypes.stream().map(TokenType::getDescription).toList().toString());
+        assertEquals(64, tokenTypes.size());
+        assertEquals(7, new HashSet<>(tokenTypes.stream().filter(DynamicMetamodelTokenType.class::isInstance).toList()).size());
 
         var bookstoreTokens = TokenUtils.tokenTypesByFile(result, TEST_SUBJECTS[0]);
         var bookstoreRenamedTokens = TokenUtils.tokenTypesByFile(result, TEST_SUBJECTS[2]);

languages/emf-metamodel/src/main/java/de/jplag/emf/MetamodelToken.java

@@ -5,35 +5,33 @@ import java.util.Optional;
 import org.eclipse.emf.ecore.EObject;
 
 import de.jplag.Token;
-import de.jplag.TokenConstants;
+import de.jplag.TokenType;
 
 /**
  * EMF metamodel token.
  * @author Timur Saglam
  */
-public class MetamodelToken extends Token implements MetamodelTokenConstants {
+public class MetamodelToken extends Token {
 
     private final Optional<EObject> eObject;
 
     /**
      * Creates an Ecore metamodel token that corresponds to an EObject.
-     * @param type is the corresponding ID of the {@link TokenConstants}.
+     * @param type is the type of the token.
      * @param file is the name of the source model file.
      * @param eObject is the corresponding eObject in the model from which this token was extracted.
      */
-    public MetamodelToken(int type, String file, EObject eObject) {
-        super(type, file, -1);
-        this.eObject = Optional.of(eObject);
+    public MetamodelToken(MetamodelTokenType type, String file, EObject eObject) {
+        this(type, file, NO_VALUE, NO_VALUE, NO_VALUE, Optional.of(eObject));
     }
 
     /**
      * Creates an Ecore metamodel token.
-     * @param type is the corresponding ID of the {@link TokenConstants}.
+     * @param type is the type of the token.
      * @param file is the name of the source model file.
      */
-    public MetamodelToken(int type, String file) {
-        super(type, file, -1);
-        this.eObject = Optional.empty();
+    public MetamodelToken(TokenType type, String file) {
+        this(type, file, NO_VALUE, NO_VALUE, NO_VALUE, Optional.empty());
     }
 
     /**
@@ -45,7 +43,7 @@ public class MetamodelToken extends Token implements MetamodelTokenConstants {
      * @param length is the length of the token in the source code.
      * @param eObject is the corresponding eObject in the model from which this token was extracted
      */
-    public MetamodelToken(int type, String file, int line, int column, int length, Optional<EObject> eObject) {
+    public MetamodelToken(TokenType type, String file, int line, int column, int length, Optional<EObject> eObject) {
         super(type, file, line, column, length);
         this.eObject = eObject;
     }
@@ -56,29 +54,4 @@ public class MetamodelToken extends Token implements MetamodelTokenConstants {
     public Optional<EObject> getEObject() {
         return eObject;
     }
-
-    @Override
-    protected String type2string() {
-        return switch (type) {
-            case PACKAGE -> "EPackage";
-            case ANNOTATION -> "EAnnotation";
-            case CLASS -> "EClass";
-            case DATATYPE -> "EDatatype";
-            case ENUM -> "EEnum";
-            case ENUM_LITERAL -> "EEnumLiteral";
-            case OPERATION -> "EOperation";
-            case REFERENCE -> "EReference";
-            case ATTRIBUTE -> "EAttribute";
-            case PARAMETER -> "EParameter";
-            case INTERFACE -> "EClass (Interface)";
-            case SUPER_TYPE -> "ESuperType";
-            case ID_ATTRIBUTE -> "EAttribute (ID)";
-            case CONTAINMENT -> "EReference (Containment)";
-            case ABSTRACT_CLASS -> "EClass (Abstract)";
-            case RETURN_TYPE -> "EClassifier (Return Type)";
-            case THROWS_DECLARATION -> "EClassifier (Exception)";
-            case FILE_END -> "(EOF)";
-            default -> "<UNKNOWN" + type + ">";
-        };
-    }
 }

languages/emf-metamodel/src/main/java/de/jplag/emf/MetamodelTokenConstants.java

@@ -1,35 +0,0 @@
-package de.jplag.emf;
-
-import de.jplag.TokenConstants;
-
-/**
- * Ecore meta-metamodel token constants. They define which tokens are extracted from a metamodel.
- * @author Timur Saglam
- */
-public interface MetamodelTokenConstants extends TokenConstants {
-
-    int PACKAGE = 2;
-    int ANNOTATION = 3;
-    int CLASS = 4;
-    int DATATYPE = 5;
-    int ENUM = 6;
-    int ENUM_LITERAL = 7;
-    int OPERATION = 8;
-    int REFERENCE = 9;
-    int ATTRIBUTE = 10;
-    int PARAMETER = 11;
-    int INTERFACE = 12;
-    int SUPER_TYPE = 13;
-    int ID_ATTRIBUTE = 14;
-    int CONTAINMENT = 15;
-    int ABSTRACT_CLASS = 16;
-    int RETURN_TYPE = 17;
-    int THROWS_DECLARATION = 18;
-    int TYPE_PARAMETER = 19;
-    int BOUND = 20;
-
-    /*
-     * Number of token constants:
-     */
-    int NUM_DIFF_TOKENS = 21;
-}

languages/emf-metamodel/src/main/java/de/jplag/emf/MetamodelTokenType.java

@@ -0,0 +1,38 @@
+package de.jplag.emf;
+
+import de.jplag.TokenType;
+
+/**
+ * Ecore meta-metamodel token type. Defines which tokens can be extracted from a metamodel.
+ */
+public enum MetamodelTokenType implements TokenType {
+    PACKAGE("EPackage"),
+    ANNOTATION("EAnnotation"),
+    CLASS("EClass"),
+    DATATYPE("EDatatype"),
+    ENUM("EEnum"),
+    ENUM_LITERAL("EEnumLiteral"),
+    OPERATION("EOperation"),
+    REFERENCE("EReference"),
+    ATTRIBUTE("EAttribute"),
+    PARAMETER("EParameter"),
+    INTERFACE("EClass (Interface)"),
+    SUPER_TYPE("ESuperType"),
+    ID_ATTRIBUTE("EAttribute (ID)"),
+    CONTAINMENT("EReference (Containment)"),
+    ABSTRACT_CLASS("EClass (Abstract)"),
+    RETURN_TYPE("EClassifier (Return Type"),
+    THROWS_DECLARATION("EClassifier (Exception"),
+    TYPE_PARAMETER("Type Parameter"),
+    BOUND("Bound");
+
+    private final String description;
+
+    public String getDescription() {
+        return description;
+    }
+
+    MetamodelTokenType(String description) {
+        this.description = description;
+    }
+}

languages/emf-metamodel/src/main/java/de/jplag/emf/parser/EcoreParser.java

@@ -8,9 +8,9 @@ import org.eclipse.emf.ecore.EObject;
 
 import de.jplag.AbstractParser;
 import de.jplag.Token;
-import de.jplag.TokenConstants;
 import de.jplag.emf.Language;
 import de.jplag.emf.MetamodelToken;
+import de.jplag.emf.MetamodelTokenType;
 import de.jplag.emf.util.AbstractMetamodelVisitor;
 import de.jplag.emf.util.EMFUtil;
 import de.jplag.emf.util.MetamodelTreeView;
@@ -63,7 +63,7 @@ public class EcoreParser extends AbstractParser {
                 visitor = createMetamodelVisitor();
                 visitor.visit(root);
             }
-            tokens.add(new MetamodelToken(TokenConstants.FILE_END, currentFile));
+            tokens.add(Token.fileEnd(currentFile));
             treeView.writeToFile(Language.VIEW_FILE_SUFFIX);
         }
     }
@@ -76,13 +76,13 @@ public class EcoreParser extends AbstractParser {
         return new MetamodelTokenGenerator(this);
     }
 
-    public void addToken(int type, EObject source, String prefix) {
+    public void addToken(MetamodelTokenType type, EObject source, String prefix) {
         MetamodelToken token = new MetamodelToken(type, currentFile, source);
         MetamodelToken metadataEnrichedToken = treeView.convertToMetadataEnrichedTokenAndAdd(token, visitor.getCurrentTreeDepth(), prefix);
         tokens.add(metadataEnrichedToken);
     }
 
-    public void addToken(int type, EObject source) {
+    public void addToken(MetamodelTokenType type, EObject source) {
         addToken(type, source, "");
     }
 }

languages/emf-metamodel/src/main/java/de/jplag/emf/parser/MetamodelTokenGenerator.java

@@ -1,24 +1,6 @@
 package de.jplag.emf.parser;
 
-import static de.jplag.emf.MetamodelTokenConstants.ABSTRACT_CLASS;
-import static de.jplag.emf.MetamodelTokenConstants.ANNOTATION;
-import static de.jplag.emf.MetamodelTokenConstants.ATTRIBUTE;
-import static de.jplag.emf.MetamodelTokenConstants.BOUND;
-import static de.jplag.emf.MetamodelTokenConstants.CLASS;
-import static de.jplag.emf.MetamodelTokenConstants.CONTAINMENT;
-import static de.jplag.emf.MetamodelTokenConstants.DATATYPE;
-import static de.jplag.emf.MetamodelTokenConstants.ENUM;
-import static de.jplag.emf.MetamodelTokenConstants.ENUM_LITERAL;
-import static de.jplag.emf.MetamodelTokenConstants.ID_ATTRIBUTE;
-import static de.jplag.emf.MetamodelTokenConstants.INTERFACE;
-import static de.jplag.emf.MetamodelTokenConstants.OPERATION;
-import static de.jplag.emf.MetamodelTokenConstants.PACKAGE;
-import static de.jplag.emf.MetamodelTokenConstants.PARAMETER;
-import static de.jplag.emf.MetamodelTokenConstants.REFERENCE;
-import static de.jplag.emf.MetamodelTokenConstants.RETURN_TYPE;
-import static de.jplag.emf.MetamodelTokenConstants.SUPER_TYPE;
-import static de.jplag.emf.MetamodelTokenConstants.THROWS_DECLARATION;
-import static de.jplag.emf.MetamodelTokenConstants.TYPE_PARAMETER;
+import static de.jplag.emf.MetamodelTokenType.*;
 
 import org.eclipse.emf.ecore.EAnnotation;
 import org.eclipse.emf.ecore.EAttribute;

languages/emf-metamodel/src/main/java/de/jplag/emf/util/MetamodelTreeView.java

@@ -55,7 +55,7 @@ public class MetamodelTreeView {
                 viewBuilder.append(System.lineSeparator());
             }
 
-            String tokenText = token.toString();
+            String tokenText = token.getType().getDescription();
             if (eObject instanceof ENamedElement element) {
                 tokenText = element.getName() + " : " + tokenText;
             }

languages/emf-metamodel/src/test/java/de/jplag/emf/MinimalMetamodelTest.java

@@ -1,11 +1,12 @@
 package de.jplag.emf;
 
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertIterableEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.File;
-import java.lang.reflect.Field;
 import java.nio.file.Path;
-import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 
@@ -17,6 +18,7 @@ import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
 import de.jplag.TokenPrinter;
+import de.jplag.TokenType;
 import de.jplag.testutils.FileUtil;
 import de.jplag.testutils.TokenUtils;
 
@@ -41,12 +43,10 @@ class MinimalMetamodelTest {
         List<Token> result = frontend.parse(baseDirectory, TEST_SUBJECTS);
 
         logger.debug(TokenPrinter.printTokens(result, baseDirectory, Optional.of(Language.VIEW_FILE_SUFFIX)));
-        Field[] fields = MetamodelTokenConstants.class.getFields();
-        var constants = Arrays.stream(fields).map(Field::getName).filter(it -> !it.equals("NUM_DIFF_TOKENS")).toList();
-        logger.info(("Handcrafted token set: " + constants));
-        logger.info("Parsed tokens: " + result.toString());
-        assertEquals(21, constants.size());
-        assertEquals(43, result.size());
+        List<TokenType> tokenTypes = result.stream().map(Token::getType).toList();
+        logger.info("Parsed token types: " + tokenTypes.stream().map(TokenType::getDescription).toList().toString());
+        assertEquals(43, tokenTypes.size());
+        assertEquals(10, new HashSet<>(tokenTypes).size());
 
         var bookstoreTokens = TokenUtils.tokenTypesByFile(result, TEST_SUBJECTS[0]);
         var bookstoreRenamedTokens = TokenUtils.tokenTypesByFile(result, TEST_SUBJECTS[2]);

languages/golang/README.md

@@ -15,7 +15,7 @@ If the grammar is updated to a more recent<a href="#footnote-1"><sup>1</sup></a>
 ### Token Extraction
 
 The choice of tokens is intended to be similar to the Java or C# frontends. Specifically, among others, it includes a range of nesting structures (class and method declarations, control flow expressions) as well as variable declaration, object creation, assignment, and control flow altering keywords. <br>
-Blocks are distinguished by their context, i.e. there are separate `TokenConstants` for `if` blocks, `for` blocks, class bodies, method bodies, array constructors, and the like.
+Blocks are distinguished by their context, i.e. there are separate `TokenTypes` for `if` blocks, `for` blocks, class bodies, method bodies, array constructors, and the like.
 
 More syntactic elements of Go may turn out to be helpful to include in the future, especially those that are newly introduced.
 

languages/golang/src/main/java/de/jplag/golang/GoParserAdapter.java

@@ -14,6 +14,7 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
 import de.jplag.Token;
+import de.jplag.TokenType;
 import de.jplag.golang.grammar.GoLexer;
 import de.jplag.golang.grammar.GoParser;
 
@@ -27,6 +28,7 @@ public class GoParserAdapter extends AbstractParser {
             if (!parseFile(directory, file)) {
                 errors++;
             }
+            tokens.add(Token.fileEnd(file));
         }
         return tokens;
     }
@@ -55,7 +57,7 @@ public class GoParserAdapter extends AbstractParser {
         return true;
     }
 
-    public void addToken(int tokenType, int line, int column, int length) {
-        tokens.add(new GoToken(tokenType, currentFile, line, column, length));
+    public void addToken(TokenType tokenType, int line, int column, int length) {
+        tokens.add(new Token(tokenType, currentFile, line, column, length));
     }
 }

languages/golang/src/main/java/de/jplag/golang/GoToken.java

@@ -1,115 +0,0 @@
-package de.jplag.golang;
-
-import static de.jplag.golang.GoTokenConstants.*;
-
-import de.jplag.Token;
-
-public class GoToken extends Token {
-
-    public GoToken(int type, String file, int line, int column, int length) {
-        super(type, file, line, column, length);
-    }
-
-    @Override
-    protected String type2string() {
-        return switch (type) {
-            case FILE_END -> "<EOF>";
-            case SEPARATOR_TOKEN -> "---------";
-
-            case PACKAGE -> "PACKAGE";
-            case IMPORT_CLAUSE -> "IMPORT";
-            case IMPORT_CLAUSE_BEGIN -> "IMPORT(";
-            case IMPORT_CLAUSE_END -> ")IMPORT";
-            case IMPORT_DECLARATION -> "IMPORT_DECL";
-
-            case ARRAY_BODY_BEGIN -> "ARRAY{";
-            case ARRAY_BODY_END -> "}ARRAY";
-
-            case STRUCT_DECLARATION -> "STRUCT";
-            case STRUCT_BODY_BEGIN -> "STRUCT{";
-            case STRUCT_BODY_END -> "}STRUCT";
-
-            case INTERFACE_DECLARATION -> "INTERFACE";
-            case INTERFACE_BLOCK_BEGIN -> "INTERFACE{";
-            case INTERFACE_BLOCK_END -> "}INTERFACE";
-
-            case INTERFACE_METHOD -> "INTERFACE_METHOD";
-            case TYPE_CONSTRAINT -> "TYPE_CONSTRAINT";
-            case TYPE_ASSERTION -> "TYPE_ASSERTION";
-
-            case MAP_BODY_BEGIN -> "MAP{";
-            case MAP_BODY_END -> "}MAP";
-
-            case SLICE_BODY_BEGIN -> "SLICE{";
-            case SLICE_BODY_END -> "}SLICE";
-
-            case NAMED_TYPE_BODY_BEGIN -> "CTYPE{";
-            case NAMED_TYPE_BODY_END -> "}CTYPE";
-
-            case MEMBER_DECLARATION -> "FIELD";
-
-            // Functions and Methods
-
-            case FUNCTION_DECLARATION -> "FUNC";
-            case RECEIVER -> "RECEIVER";
-            case FUNCTION_PARAMETER -> "PARAM";
-            case FUNCTION_BODY_BEGIN -> "FUNC{";
-            case FUNCTION_BODY_END -> "}FUNC";
-
-            // Control flow statements
-
-            case IF_STATEMENT -> "IF";
-
-            case IF_BLOCK_BEGIN -> "IF{";
-            case IF_BLOCK_END -> "}IF";
-            case ELSE_BLOCK_BEGIN -> "ELSE{";
-            case ELSE_BLOCK_END -> "}ELSE";
-            case FOR_STATEMENT -> "FOR";
-            case FOR_BLOCK_BEGIN -> "FOR{";
-            case FOR_BLOCK_END -> "}FOR";
-            case SWITCH_STATEMENT -> "SWITCH";
-            case SWITCH_BLOCK_BEGIN -> "SWITCH{";
-            case SWITCH_BLOCK_END -> "}SWITCH";
-            case SWITCH_CASE -> "CASE";
-            case SELECT_STATEMENT -> "SELECT";
-            case SELECT_BLOCK_BEGIN -> "SELECT{";
-            case SELECT_BLOCK_END -> "}SELECT";
-            case CASE_BLOCK_BEGIN -> "CASE{";
-            case CASE_BLOCK_END -> "}CASE";
-
-            // Statements
-
-            case VARIABLE_DECLARATION -> "VAR_DECL";
-            case FUNCTION_LITERAL -> "FUNC_LIT";
-            case ASSIGNMENT -> "ASSIGN";
-            case SEND_STATEMENT -> "SEND";
-            case RECEIVE_STATEMENT -> "RECV";
-            case INVOCATION -> "INVOC";
-            case ARGUMENT -> "ARG";
-            case STATEMENT_BLOCK_BEGIN -> "INNER{";
-            case STATEMENT_BLOCK_END -> "}INNER";
-
-            // Object Creation
-
-            case ARRAY_ELEMENT -> "ARRAY_ELEM";
-            case MAP_ELEMENT -> "MAP_ELEM";
-            case SLICE_ELEMENT -> "SLICE_ELEM";
-            case NAMED_TYPE_ELEMENT -> "CTYPE_ELEM";
-            case ARRAY_CONSTRUCTOR -> "ARRAY()";
-            case SLICE_CONSTRUCTOR -> "SLICE()";
-            case MAP_CONSTRUCTOR -> "MAP()";
-            case NAMED_TYPE_CONSTRUCTOR -> "CTYPE()";
-
-            // Control Flow Keywords
-
-            case RETURN -> "RETURN";
-            case BREAK -> "BREAK";
-            case CONTINUE -> "CONTINUE";
-            case FALLTHROUGH -> "FALLTHROUGH";
-            case GOTO -> "GOTO";
-            case GO -> "GO";
-            case DEFER -> "DEFER";
-            default -> "<UNKNOWN%d>".formatted(type);
-        };
-    }
-}

languages/golang/src/main/java/de/jplag/golang/GoTokenConstants.java

@@ -1,96 +0,0 @@
-package de.jplag.golang;
-
-import de.jplag.TokenConstants;
-
-public interface GoTokenConstants extends TokenConstants {
-
-    // TOP LEVEL STRUCTURES
-
-    int PACKAGE = 2;
-    int IMPORT_CLAUSE = 3;
-    int IMPORT_CLAUSE_BEGIN = 4;
-    int IMPORT_CLAUSE_END = 5;
-    int IMPORT_DECLARATION = 6;
-    int ARRAY_BODY_BEGIN = 7;
-    int ARRAY_BODY_END = 8;
-    int STRUCT_DECLARATION = 9;
-    int STRUCT_BODY_BEGIN = 10;
-    int STRUCT_BODY_END = 11;
-    int INTERFACE_DECLARATION = 12;
-    int INTERFACE_BLOCK_BEGIN = 13;
-    int INTERFACE_BLOCK_END = 14;
-    int INTERFACE_METHOD = 15;
-    int TYPE_CONSTRAINT = 16;
-    int TYPE_ASSERTION = 17;
-    int MAP_BODY_BEGIN = 18;
-    int MAP_BODY_END = 19;
-    int SLICE_BODY_BEGIN = 20;
-    int SLICE_BODY_END = 21;
-    int NAMED_TYPE_BODY_BEGIN = 22;
-    int NAMED_TYPE_BODY_END = 23;
-    int MEMBER_DECLARATION = 24;
-
-    // FUNCTIONS AND METHODS
-
-    int FUNCTION_DECLARATION = 25;
-    int RECEIVER = 26;
-    int FUNCTION_PARAMETER = 27;
-    int FUNCTION_BODY_BEGIN = 28;
-    int FUNCTION_BODY_END = 29;
-
-    // CONTROL FLOW STATEMENTS
-
-    int IF_STATEMENT = 30;
-    int IF_BLOCK_BEGIN = 31;
-    int IF_BLOCK_END = 32;
-    int ELSE_BLOCK_BEGIN = 33;
-    int ELSE_BLOCK_END = 34;
-    int FOR_STATEMENT = 35;
-    int FOR_BLOCK_BEGIN = 36;
-    int FOR_BLOCK_END = 37;
-    int SWITCH_STATEMENT = 38;
-    int SWITCH_BLOCK_BEGIN = 39;
-    int SWITCH_BLOCK_END = 40;
-    int SWITCH_CASE = 41;
-    int SELECT_STATEMENT = 42;
-    int SELECT_BLOCK_BEGIN = 43;
-    int SELECT_BLOCK_END = 44;
-    int CASE_BLOCK_BEGIN = 45;
-    int CASE_BLOCK_END = 46;
-
-    // STATEMENTS
-
-    int VARIABLE_DECLARATION = 47;
-    int FUNCTION_LITERAL = 48;
-    int ASSIGNMENT = 49;
-    int SEND_STATEMENT = 50;
-    int RECEIVE_STATEMENT = 51;
-    int INVOCATION = 52;
-    int ARGUMENT = 53;
-    int STATEMENT_BLOCK_BEGIN = 54;
-    int STATEMENT_BLOCK_END = 55;
-
-    // OBJECT CREATION
-
-    int ARRAY_ELEMENT = 56;
-    int MAP_ELEMENT = 57;
-    int SLICE_ELEMENT = 58;
-    int NAMED_TYPE_ELEMENT = 59;
-    int ARRAY_CONSTRUCTOR = 60;
-    int MAP_CONSTRUCTOR = 61;
-    int SLICE_CONSTRUCTOR = 62;
-    int NAMED_TYPE_CONSTRUCTOR = 63;
-
-    // CONTROL FLOW KEYWORDS
-
-    int RETURN = 64;
-    int BREAK = 65;
-    int CONTINUE = 66;
-    int FALLTHROUGH = 67;
-    int GOTO = 68;
-    int GO = 69;
-    int DEFER = 70;
-
-    int NUM_DIFF_TOKENS = 71;
-
-}

languages/golang/src/main/java/de/jplag/golang/GoTokenType.java

@@ -0,0 +1,102 @@
+package de.jplag.golang;
+
+import de.jplag.TokenType;
+
+public enum GoTokenType implements TokenType {
+    // TOP LEVEL STRUCTURES
+
+    PACKAGE("PACKAGE"),
+    IMPORT_CLAUSE("IMPORT"),
+    IMPORT_CLAUSE_BEGIN("IMPORT("),
+    IMPORT_CLAUSE_END(")IMPORT"),
+    IMPORT_DECLARATION("IMPORT_DECL"),
+    ARRAY_BODY_BEGIN("ARRAY{"),
+    ARRAY_BODY_END("}ARRAY"),
+    STRUCT_DECLARATION("STRUCT"),
+    STRUCT_BODY_BEGIN("STRUCT{"),
+    STRUCT_BODY_END("}STRUCT"),
+    INTERFACE_DECLARATION("INTERFACE"),
+    INTERFACE_BLOCK_BEGIN("INTERFACE{"),
+    INTERFACE_BLOCK_END("}INTERFACE"),
+    INTERFACE_METHOD("INTERFACE_METHOD"),
+    TYPE_CONSTRAINT("TYPE_CONSTRAINT"),
+    TYPE_ASSERTION("TYPE_ASSERTION"),
+    MAP_BODY_BEGIN("MAP{"),
+    MAP_BODY_END("}MAP"),
+    SLICE_BODY_BEGIN("SLICE{"),
+    SLICE_BODY_END("}SLICE"),
+    NAMED_TYPE_BODY_BEGIN("CTYPE{"),
+    NAMED_TYPE_BODY_END("}CTYPE"),
+    MEMBER_DECLARATION("FIELD"),
+
+    // FUNCTIONS AND METHODS
+
+    FUNCTION_DECLARATION("FUNC"),
+    RECEIVER("RECEIVER"),
+    FUNCTION_PARAMETER("PARAM"),
+    FUNCTION_BODY_BEGIN("FUNC{"),
+    FUNCTION_BODY_END("}FUNC"),
+
+    // CONTROL FLOW STATEMENTS
+
+    IF_STATEMENT("IF"),
+    IF_BLOCK_BEGIN("IF{"),
+    IF_BLOCK_END("}IF"),
+    ELSE_BLOCK_BEGIN("ELSE{"),
+    ELSE_BLOCK_END("}ELSE"),
+    FOR_STATEMENT("FOR"),
+    FOR_BLOCK_BEGIN("FOR{"),
+    FOR_BLOCK_END("}FOR"),
+    SWITCH_STATEMENT("SWITCH"),
+    SWITCH_BLOCK_BEGIN("SWITCH{"),
+    SWITCH_BLOCK_END("}SWITCH"),
+    SWITCH_CASE("CASE"),
+    SELECT_STATEMENT("SELECT"),
+    SELECT_BLOCK_BEGIN("SELECT{"),
+    SELECT_BLOCK_END("}SELECT"),
+    CASE_BLOCK_BEGIN("CASE{"),
+    CASE_BLOCK_END("}CASE"),
+
+    // STATEMENTS
+
+    VARIABLE_DECLARATION("VAR_DECL"),
+    FUNCTION_LITERAL("FUNC_LIT"),
+    ASSIGNMENT("ASSIGN"),
+    SEND_STATEMENT("SEND"),
+    RECEIVE_STATEMENT("RECV"),
+    INVOCATION("INVOC"),
+    ARGUMENT("ARG"),
+    STATEMENT_BLOCK_BEGIN("INNER{"),
+    STATEMENT_BLOCK_END("}INNER"),
+
+    // OBJECT CREATION
+
+    ARRAY_ELEMENT("ARRAY_ELEM"),
+    MAP_ELEMENT("MAP_ELEM"),
+    SLICE_ELEMENT("SLICE_ELEM"),
+    NAMED_TYPE_ELEMENT("CTYPE_ELEM"),
+    ARRAY_CONSTRUCTOR("ARRAY()"),
+    MAP_CONSTRUCTOR("MAP()"),
+    SLICE_CONSTRUCTOR("SLICE()"),
+    NAMED_TYPE_CONSTRUCTOR("CTYPE()"),
+
+    // CONTROL FLOW KEYWORDS
+
+    RETURN("RETURN"),
+    BREAK("BREAK"),
+    CONTINUE("CONTINUE"),
+    FALLTHROUGH("FALLTHROUGH"),
+    GOTO("GOTO"),
+    GO("GO"),
+    DEFER("DEFER");
+
+    private final String description;
+
+    public String getDescription() {
+        return this.description;
+    }
+
+    GoTokenType(String description) {
+        this.description = description;
+    }
+}

languages/golang/src/main/java/de/jplag/golang/JPlagGoListener.java

@@ -1,18 +1,18 @@
 package de.jplag.golang;
 
-import static de.jplag.golang.GoTokenConstants.*;
+import static de.jplag.golang.GoTokenType.*;
 
 import java.util.*;
 
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.tree.TerminalNode;
 
+import de.jplag.TokenType;
 import de.jplag.golang.grammar.GoParser;
 import de.jplag.golang.grammar.GoParserBaseListener;
 
 public class JPlagGoListener extends GoParserBaseListener {
 
-    public static final int NONE = -1;
     private final GoParserAdapter parserAdapter;
     private final Deque<GoBlockContext> blockContexts;
 
@@ -26,7 +26,7 @@ public class JPlagGoListener extends GoParserBaseListener {
      * @param tokenType the custom token type that occurred.
      * @param token the corresponding grammar's token
      */
-    private void transformToken(int tokenType, Token token) {
+    private void transformToken(TokenType tokenType, Token token) {
         parserAdapter.addToken(tokenType, token.getLine(), token.getCharPositionInLine() + 1, token.getText().length());
     }
 
@@ -37,7 +37,7 @@ public class JPlagGoListener extends GoParserBaseListener {
      * @param start the first Token of the context
      * @param end the last Token of the context
      */
-    private void transformToken(int tokenType, Token start, Token end) {
+    private void transformToken(GoTokenType tokenType, Token start, Token end) {
         parserAdapter.addToken(tokenType, start.getLine(), start.getCharPositionInLine() + 1, end.getStopIndex() - start.getStartIndex() + 1);
     }
 
@@ -343,9 +343,8 @@ public class JPlagGoListener extends GoParserBaseListener {
 
     @Override
     public void enterKeyedElement(GoParser.KeyedElementContext context) {
-        int tokenType = getCurrentContext().getElement();
-        transformToken(tokenType, context.getStart(), context.getStop());
-
+        Optional<GoTokenType> tokenType = getCurrentContext().getElement();
+        tokenType.ifPresent(type -> transformToken(type, context.getStart(), context.getStop()));
         super.enterKeyedElement(context);
     }
 
@@ -474,11 +473,6 @@ public class JPlagGoListener extends GoParserBaseListener {
             }
             case "{" -> transformToken(getCurrentContext().getBegin(), token);
             case "}" -> transformToken(getCurrentContext().getEnd(), token);
-            case "<EOF>" -> {
-                if (node.getParent() instanceof GoParser.SourceFileContext) {
-                    transformToken(FILE_END, token);
-                }
-            }
             default -> {
                 // do nothing.
             }
@@ -494,41 +488,41 @@ public class JPlagGoListener extends GoParserBaseListener {
      * required to be able to assign the correct token types for each block.
      */
     private enum GoBlockContext {
-        ARRAY_BODY(ARRAY_BODY_BEGIN, ARRAY_BODY_END, ARRAY_ELEMENT),
-        STRUCT_BODY(STRUCT_BODY_BEGIN, STRUCT_BODY_END, MEMBER_DECLARATION),
-        MAP_BODY(MAP_BODY_BEGIN, MAP_BODY_END, MAP_ELEMENT),
-        SLICE_BODY(SLICE_BODY_BEGIN, SLICE_BODY_END, SLICE_ELEMENT),
-        NAMED_TYPE_BODY(NAMED_TYPE_BODY_BEGIN, NAMED_TYPE_BODY_END, NAMED_TYPE_ELEMENT),
-        FUNCTION_BODY(FUNCTION_BODY_BEGIN, FUNCTION_BODY_END, NONE),
-
-        IF_BLOCK(IF_BLOCK_BEGIN, IF_BLOCK_END, NONE),
-        ELSE_BLOCK(ELSE_BLOCK_BEGIN, ELSE_BLOCK_END, NONE),
-        FOR_BLOCK(FOR_BLOCK_BEGIN, FOR_BLOCK_END, NONE),
-        SWITCH_BLOCK(SWITCH_BLOCK_BEGIN, SWITCH_BLOCK_END, NONE),
-        SELECT_CONTEXT(SELECT_BLOCK_BEGIN, SELECT_BLOCK_END, NONE),
-        STATEMENT_BLOCK(STATEMENT_BLOCK_BEGIN, STATEMENT_BLOCK_END, NONE),
-        CASE_BLOCK(CASE_BLOCK_BEGIN, CASE_BLOCK_END, NONE),
-        INTERFACE_BODY(INTERFACE_BLOCK_BEGIN, INTERFACE_BLOCK_END, NONE);
-
-        private final int beginTokenType;
-        private final int endTokenType;
-        private final int elementTokenType;
-
-        GoBlockContext(int beginTokenType, int endTokenType, int elementTokenType) {
+        ARRAY_BODY(ARRAY_BODY_BEGIN, ARRAY_BODY_END, Optional.of(ARRAY_ELEMENT)),
+        STRUCT_BODY(STRUCT_BODY_BEGIN, STRUCT_BODY_END, Optional.of(MEMBER_DECLARATION)),
+        MAP_BODY(MAP_BODY_BEGIN, MAP_BODY_END, Optional.of(MAP_ELEMENT)),
+        SLICE_BODY(SLICE_BODY_BEGIN, SLICE_BODY_END, Optional.of(SLICE_ELEMENT)),
+        NAMED_TYPE_BODY(NAMED_TYPE_BODY_BEGIN, NAMED_TYPE_BODY_END, Optional.of(NAMED_TYPE_ELEMENT)),
+        FUNCTION_BODY(FUNCTION_BODY_BEGIN, FUNCTION_BODY_END, Optional.empty()),
+
+        IF_BLOCK(IF_BLOCK_BEGIN, IF_BLOCK_END, Optional.empty()),
+        ELSE_BLOCK(ELSE_BLOCK_BEGIN, ELSE_BLOCK_END, Optional.empty()),
+        FOR_BLOCK(FOR_BLOCK_BEGIN, FOR_BLOCK_END, Optional.empty()),
+        SWITCH_BLOCK(SWITCH_BLOCK_BEGIN, SWITCH_BLOCK_END, Optional.empty()),
+        SELECT_CONTEXT(SELECT_BLOCK_BEGIN, SELECT_BLOCK_END, Optional.empty()),
+        STATEMENT_BLOCK(STATEMENT_BLOCK_BEGIN, STATEMENT_BLOCK_END, Optional.empty()),
+        CASE_BLOCK(CASE_BLOCK_BEGIN, CASE_BLOCK_END, Optional.empty()),
+        INTERFACE_BODY(INTERFACE_BLOCK_BEGIN, INTERFACE_BLOCK_END, Optional.empty());
+
+        private final GoTokenType beginTokenType;
+        private final GoTokenType endTokenType;
+        private final Optional<GoTokenType> elementTokenType;
+
+        GoBlockContext(GoTokenType beginTokenType, GoTokenType endTokenType, Optional<GoTokenType> elementTokenType) {
             this.beginTokenType = beginTokenType;
             this.endTokenType = endTokenType;
             this.elementTokenType = elementTokenType;
         }
 
-        int getBegin() {
+        GoTokenType getBegin() {
             return this.beginTokenType;
         }
 
-        int getEnd() {
+        GoTokenType getEnd() {
             return this.endTokenType;
         }
 
-        public int getElement() {
+        public Optional<GoTokenType> getElement() {
             return this.elementTokenType;
         }
     }

languages/golang/src/test/java/de/jplag/golang/GoFrontendTest.java

@@ -1,6 +1,6 @@
 package de.jplag.golang;
 
-import static de.jplag.golang.GoTokenTestUtils.getDummyToken;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
 import java.io.File;
@@ -8,6 +8,7 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 import java.util.OptionalInt;
@@ -19,8 +20,8 @@ import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import de.jplag.SharedTokenType;
 import de.jplag.Token;
-import de.jplag.TokenConstants;
 import de.jplag.TokenPrinter;
 
 class GoFrontendTest {
@@ -68,21 +69,6 @@ class GoFrontendTest {
         }
     }
 
-    /**
-     * Confirms that every type of GoToken has a Sting representation associated to it.
-     */
-    @Test
-    void testTokenToString() {
-        var missingTokens = IntStream.range(0, GoTokenConstants.NUM_DIFF_TOKENS).mapToObj(GoTokenTestUtils::getDummyToken)
-                .filter(token -> token.type2string().contains("UNKNOWN")).toList();
-
-        if (!missingTokens.isEmpty()) {
-            var typeList = missingTokens.stream().map(Token::getType).map(Object::toString).collect(Collectors.joining(", "));
-            fail("Found token types with no string representation: %s".formatted(typeList));
-        }
-
-    }
-
     /**
      * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
      * @param fileName a code sample file name
@@ -102,7 +88,7 @@ class GoFrontendTest {
         // All lines that contain code
         var codeLines = getCodeLines(lines);
         // All lines that contain a token
-        var tokenLines = tokens.stream().map(Token::getLine).distinct().toList();
+        var tokenLines = tokens.stream().mapToInt(Token::getLine).distinct().boxed().toList();
 
         if (codeLines.size() > tokenLines.size()) {
             List<Integer> missedLinesIndices = new ArrayList<>(codeLines);
@@ -156,18 +142,13 @@ class GoFrontendTest {
      * @param fileName The file name of the complete code example
      */
     private void testTokenCoverage(List<Token> tokens, String fileName) {
-        var foundTokens = tokens.stream().parallel().map(Token::getType).sorted().distinct().toList();
-
-        // Exclude SEPARATOR_TOKEN, as it does not occur
-        var missingTokenTypes = IntStream.range(0, GoTokenConstants.NUM_DIFF_TOKENS).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).boxed()
-                .collect(Collectors.toList());
-        missingTokenTypes.removeAll(foundTokens);
-
-        if (!missingTokenTypes.isEmpty()) {
-            String missingTypesList = missingTokenTypes.stream().map(type -> getDummyToken(type).type2string()).collect(Collectors.joining(", "));
-            fail("Some Token types were not found in the complete code example '%s': %s.".formatted(fileName, missingTypesList));
-        }
-
+        var annotatedTokens = tokens.stream().map(Token::getType).collect(Collectors.toSet());
+        assertTrue(annotatedTokens.contains(SharedTokenType.FILE_END));
+        var annotatedGoTokens = annotatedTokens.stream().filter(GoTokenType.class::isInstance).collect(Collectors.toSet());
+        var allGoTokens = GoTokenType.values();
+        var missingGoTokens = Arrays.stream(allGoTokens).filter(token -> !annotatedGoTokens.contains(token)).toList();
+        assertTrue(missingGoTokens.isEmpty(), "The following go tokens are missing in the code example '%s':\n".formatted(fileName)
+                + String.join("\n", missingGoTokens.stream().map(GoTokenType::getDescription).toList()));
     }
 
 }

languages/golang/src/test/java/de/jplag/golang/GoTokenTestUtils.java

@@ -1,20 +0,0 @@
-package de.jplag.golang;
-
-public class GoTokenTestUtils {
-
-    private static final int NOT_SET = -1;
-    private static final String EMPTY_STRING = "";
-
-    private GoTokenTestUtils() {
-        // Utility class, should not be instantiated.
-    }
-
-    public static GoToken getDummyToken(int type) {
-        return new GoToken(type, EMPTY_STRING, NOT_SET, NOT_SET, NOT_SET);
-    }
-
-    public static GoToken getDummyToken(int type, String fileName) {
-        return new GoToken(type, fileName, NOT_SET, NOT_SET, NOT_SET);
-    }
-
-}

languages/java/src/main/java/de/jplag/java/JavaToken.java

@@ -1,166 +0,0 @@
-package de.jplag.java;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import de.jplag.Token;
-import de.jplag.TokenConstants;
-
-public class JavaToken extends Token implements JavaTokenConstants {
-
-    private final Logger logger = LoggerFactory.getLogger(JavaToken.class);
-
-    public JavaToken(int type, String file, int line, int column, int length) {
-        super(type, file, line, column, length);
-    }
-
-    @Override
-    protected String type2string() {
-        switch (type) {
-            case TokenConstants.FILE_END:
-                return "EOF";
-            case TokenConstants.SEPARATOR_TOKEN:
-                return "METHOD_SEPARATOR";
-            case J_PACKAGE:
-                return "PACKAGE";
-            case J_IMPORT:
-                return "IMPORT";
-            case J_CLASS_BEGIN:
-                return "CLASS{";
-            case J_CLASS_END:
-                return "}CLASS";
-            case J_METHOD_BEGIN:
-                return "METHOD{";
-            case J_METHOD_END:
-                return "}METHOD";
-            case J_VARDEF:
-                return "VARDEF";
-            case J_SYNC_BEGIN:
-                return "SYNC{";
-            case J_SYNC_END:
-                return "}SYNC";
-            case J_DO_BEGIN:
-                return "DO{";
-            case J_DO_END:
-                return "}DO";
-            case J_WHILE_BEGIN:
-                return "WHILE{";
-            case J_WHILE_END:
-                return "}WHILE";
-            case J_FOR_BEGIN:
-                return "FOR{";
-            case J_FOR_END:
-                return "}FOR";
-            case J_SWITCH_BEGIN:
-                return "SWITCH{";
-            case J_SWITCH_END:
-                return "}SWITCH";
-            case J_CASE:
-                return "CASE";
-            case J_TRY_BEGIN:
-                return "TRY{";
-            case J_CATCH_BEGIN:
-                return "CATCH{";
-            case J_CATCH_END:
-                return "}CATCH";
-            case J_FINALLY:
-                return "FINALLY";
-            case J_IF_BEGIN:
-                return "IF{";
-            case J_ELSE:
-                return "ELSE";
-            case J_IF_END:
-                return "}IF";
-            case J_COND:
-                return "COND";
-            case J_BREAK:
-                return "BREAK";
-            case J_CONTINUE:
-                return "CONTINUE";
-            case J_RETURN:
-                return "RETURN";
-            case J_THROW:
-                return "THROW";
-            case J_IN_CLASS_BEGIN:
-                return "INCLASS{";
-            case J_IN_CLASS_END:
-                return "}INCLASS";
-            case J_APPLY:
-                return "APPLY";
-            case J_NEWCLASS:
-                return "NEWCLASS";
-            case J_NEWARRAY:
-                return "NEWARRAY";
-            case J_ASSIGN:
-                return "ASSIGN";
-            case J_INTERFACE_BEGIN:
-                return "INTERF{";
-            case J_INTERFACE_END:
-                return "}INTERF";
-            case J_CONSTR_BEGIN:
-                return "CONSTR{";
-            case J_CONSTR_END:
-                return "}CONSTR";
-            case J_INIT_BEGIN:
-                return "INIT{";
-            case J_INIT_END:
-                return "}INIT";
-            case J_VOID:
-                return "VOID";
-            case J_ARRAY_INIT_BEGIN:
-                return "ARRINIT{";
-            case J_ARRAY_INIT_END:
-                return "ARRINIT}";
-            case J_ENUM_BEGIN:
-                return "ENUM{";
-            case J_ENUM_CLASS_BEGIN:
-                return "ENUM_CLA";
-            case J_ENUM_END:
-                return "}ENUM";
-            case J_GENERIC:
-                return "GENERIC";
-            case J_ASSERT:
-                return "ASSERT";
-            case J_ANNO:
-                return "ANNO";
-            case J_ANNO_MARKER:
-                return "ANNOMARK";
-            case J_ANNO_M_BEGIN:
-                return "ANNO_M{";
-            case J_ANNO_M_END:
-                return "}ANNO_M";
-            case J_ANNO_T_BEGIN:
-                return "ANNO_T{";
-            case J_ANNO_T_END:
-                return "}ANNO_T";
-            case J_ANNO_C_BEGIN:
-                return "ANNO_C{";
-            case J_ANNO_C_END:
-                return "}ANNO_C";
-            case J_MODULE_BEGIN:
-                return "MODULE{";
-            case J_MODULE_END:
-                return "}MODULE";
-            case J_EXPORTS:
-                return "EXPORTS";
-            case J_PROVIDES:
-                return "PROVIDES";
-            case J_REQUIRES:
-                return "REQUIRES";
-            case J_TRY_WITH_RESOURCE:
-                return "TRY_RES";
-            case J_YIELD:
-                return "YIELD";
-            case J_DEFAULT:
-                return "DEFAULT";
-            case J_RECORD_BEGIN:
-                return "RECORD{";
-            case J_RECORD_END:
-                return "}RECORD";
-
-            default:
-                logger.error("UNKNOWN: " + type);
-                return "<UNKNOWN" + type + ">";
-        }
-    }
-}

languages/java/src/main/java/de/jplag/java/JavaTokenConstants.java

@@ -1,88 +0,0 @@
-package de.jplag.java;
-
-import de.jplag.TokenConstants;
-
-public interface JavaTokenConstants extends TokenConstants {
-
-    int J_PACKAGE = 2;                // check
-    int J_IMPORT = 3;                // check
-    int J_CLASS_BEGIN = 4;            // check
-    int J_CLASS_END = 5;            // check
-    int J_METHOD_BEGIN = 6;        // check
-    int J_METHOD_END = 7;            // check
-    int J_VARDEF = 8;                // check
-    int J_SYNC_BEGIN = 9;            // check
-    int J_SYNC_END = 10;            // check
-    int J_DO_BEGIN = 11;            // check
-    int J_DO_END = 12;                // check
-    int J_WHILE_BEGIN = 13;        // check
-    int J_WHILE_END = 14;            // check
-    int J_FOR_BEGIN = 15;            // check
-    int J_FOR_END = 16;            // check
-    int J_SWITCH_BEGIN = 17;        // check
-    int J_SWITCH_END = 18;            // check
-    int J_CASE = 19;                // check
-    int J_TRY_BEGIN = 20;            // check
-    int J_CATCH_BEGIN = 21;        // check
-    int J_CATCH_END = 22;        // check
-    int J_FINALLY = 23;            // check
-    int J_IF_BEGIN = 24;            // check
-    int J_ELSE = 25;                // check
-    int J_IF_END = 26;                // check
-    int J_COND = 27;                // check
-    int J_BREAK = 28;                // check
-    int J_CONTINUE = 29;            // check
-    int J_RETURN = 30;                // check
-    int J_THROW = 31;                // check
-    int J_IN_CLASS_BEGIN = 32;        //
-    int J_IN_CLASS_END = 33;        //
-    int J_APPLY = 34;                // check
-    int J_NEWCLASS = 35;            // check
-    int J_NEWARRAY = 36;            // check
-    int J_ASSIGN = 37;                // check
-    int J_INTERFACE_BEGIN = 38;    // check
-    int J_INTERFACE_END = 39;        // check
-    int J_CONSTR_BEGIN = 40;        //
-    int J_CONSTR_END = 41;            //
-    int J_INIT_BEGIN = 42;            // check
-    int J_INIT_END = 43;            // check
-    int J_VOID = 44;                //
-    int J_ARRAY_INIT_BEGIN = 45;    // check
-    int J_ARRAY_INIT_END = 46;    // check
-
-    // new in 1.5:
-    int J_ENUM_BEGIN = 47;            // check
-    int J_ENUM_CLASS_BEGIN = 48;    // ?? doesn't exist in JAVAC
-    int J_ENUM_END = 49;            // check
-    int J_GENERIC = 50;            // check
-    int J_ASSERT = 51;                // check
-
-    int J_ANNO = 52;                // check
-    int J_ANNO_MARKER = 53;        // ??
-    int J_ANNO_M_BEGIN = 54;        // ??
-    int J_ANNO_M_END = 55;            // ??
-    int J_ANNO_T_BEGIN = 56;        // check
-    int J_ANNO_T_END = 57;            // check
-    int J_ANNO_C_BEGIN = 58;        // ??
-    int J_ANNO_C_END = 59;            // ??
-
-    // new in 1.7
-    int J_TRY_WITH_RESOURCE = 60;    // check
-
-    // new in 1.9
-    int J_REQUIRES = 61;            // check
-    int J_PROVIDES = 62;            // check
-    int J_EXPORTS = 63;            // check
-    int J_MODULE_BEGIN = 64;       // check
-    int J_MODULE_END = 65;        // check
-
-    // new in 13
-    int J_YIELD = 66;
-
-    // new in 17
-    int J_DEFAULT = 67;
-    int J_RECORD_BEGIN = 68;
-    int J_RECORD_END = 69;
-
-    int NUM_DIFF_TOKENS = 70;	    // @formatter:on
-}

languages/java/src/main/java/de/jplag/java/JavaTokenType.java

@@ -0,0 +1,94 @@
+package de.jplag.java;
+
+import de.jplag.TokenType;
+
+public enum JavaTokenType implements TokenType {
+    J_PACKAGE("PACKAGE"), // check
+    J_IMPORT("IMPORT"), // check
+    J_CLASS_BEGIN("CLASS{"), // check
+    J_CLASS_END("}CLASS"), // check
+    J_METHOD_BEGIN("METHOD{"), // check
+    J_METHOD_END("}METHOD"), // check
+    J_VARDEF("VARDEF"), // check
+    J_SYNC_BEGIN("SYNC{"), // check
+    J_SYNC_END("}SYNC"), // check
+    J_DO_BEGIN("DO{"), // check
+    J_DO_END("}DO"), // check
+    J_WHILE_BEGIN("WHILE{"), // check
+    J_WHILE_END("}WHILE"), // check
+    J_FOR_BEGIN("FOR{"), // check
+    J_FOR_END("}FOR"), // check
+    J_SWITCH_BEGIN("SWITCH{"), // check
+    J_SWITCH_END("}SWITCH"), // check
+    J_CASE("CASE"), // check
+    J_TRY_BEGIN("TRY{"), // check
+    J_CATCH_BEGIN("CATCH{"), // check
+    J_CATCH_END("}CATCH"), // check
+    J_FINALLY("FINALLY"), // check
+    J_IF_BEGIN("IF{"), // check
+    J_ELSE("ELSE"), // check
+    J_IF_END("}IF"), // check
+    J_COND("COND"), // check
+    J_BREAK("BREAK"), // check
+    J_CONTINUE("CONTINUE"), // check
+    J_RETURN("RETURN"), // check
+    J_THROW("THROW"), // check
+    J_IN_CLASS_BEGIN("INCLASS{"), //
+    J_IN_CLASS_END("}INCLASS"), //
+    J_APPLY("APPLY"), // check
+    J_NEWCLASS("NEWCLASS"), // check
+    J_NEWARRAY("NEWARRAY"), // check
+    J_ASSIGN("ASSIGN"), // check
+    J_INTERFACE_BEGIN("INTERF{"), // check
+    J_INTERFACE_END("}INTERF"), // check
+    J_CONSTR_BEGIN("CONSTR{"), //
+    J_CONSTR_END("}CONSTR"), //
+    J_INIT_BEGIN("INIT{"), // check
+    J_INIT_END("}INIT"), // check
+    J_VOID("VOID"), //
+    J_ARRAY_INIT_BEGIN("ARRINIT{"), // check
+    J_ARRAY_INIT_END("}ARRINIT"), // check
+
+    // new in 1.5
+    J_ENUM_BEGIN("ENUM"), // check
+    J_ENUM_CLASS_BEGIN("ENUM_CLA"), // ?? doesn't exist in JAVAC
+    J_ENUM_END("}ENUM"), // check
+    J_GENERIC("GENERIC"), // check
+    J_ASSERT("ASSERT"), // check
+    J_ANNO("ANNO"), // check
+    J_ANNO_MARKER("ANNOMARK"), // ??
+    J_ANNO_M_BEGIN("ANNO_M{"), // ??
+    J_ANNO_M_END("}ANNO_M"), // ??
+    J_ANNO_T_BEGIN("ANNO_T{"), // check
+    J_ANNO_T_END("}ANNO_T"), // check
+    J_ANNO_C_BEGIN("ANNO_C{"), // ??
+    J_ANNO_C_END("}ANNO_C"), // ??
+
+    // new in 1.7
+    J_TRY_WITH_RESOURCE("TRY_RES"), // check
+
+    // new in 1.9
+    J_REQUIRES("REQUIRES"), // check
+    J_PROVIDES("PROVIDES"), // check
+    J_EXPORTS("EXPORTS"), // check
+    J_MODULE_BEGIN("MODULE{"), // check
+    J_MODULE_END("}MODULE"), // check
+
+    // new in 13
+    J_YIELD("YIELD"),
+
+    // new in 17
+    J_DEFAULT("DEFAULT"),
+    J_RECORD_BEGIN("RECORD{"),
+    J_RECORD_END("}RECORD");
+
+    private final String description;
+
+    public String getDescription() {
+        return this.description;
+    }
+
+    JavaTokenType(String description) {
+        this.description = description;
+    }
+}

languages/java/src/main/java/de/jplag/java/JavacAdapter.java

@@ -18,7 +18,7 @@ import com.sun.source.util.JavacTask;
 import com.sun.source.util.SourcePositions;
 import com.sun.source.util.Trees;
 
-import de.jplag.TokenConstants;
+import de.jplag.Token;
 
 public class JavacAdapter {
 
@@ -40,7 +40,7 @@ public class JavacAdapter {
                 final String filename = fileNameOf(directory, ast);
                 final LineMap map = ast.getLineMap();
                 ast.accept(new TokenGeneratingTreeScanner(filename, parser, map, positions, ast), null);
-                parser.add(TokenConstants.FILE_END, filename, 1, -1, -1);
+                parser.add(Token.fileEnd(filename));
             }
         } catch (IOException e) {
             parser.logger.error(e.getMessage(), e);

languages/java/src/main/java/de/jplag/java/Parser.java

@@ -7,6 +7,7 @@ import java.util.List;
 
 import de.jplag.AbstractParser;
 import de.jplag.Token;
+import de.jplag.TokenType;
 
 public class Parser extends AbstractParser {
     private List<Token> tokens;
@@ -26,8 +27,12 @@ public class Parser extends AbstractParser {
         return tokens;
     }
 
-    public void add(int type, String filename, long line, long column, long length) {
-        tokens.add(new JavaToken(type, filename, (int) line, (int) column, (int) length));
+    public void add(TokenType type, String filename, long line, long column, long length) {
+        add(new Token(type, filename, (int) line, (int) column, (int) length));
+    }
+
+    public void add(Token token) {
+        tokens.add(token);
     }
 
     public void increaseErrors() {

languages/java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java

@@ -59,21 +59,21 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
 
     /**
      * Convenience method that adds a specific token.
-     * @param tokenType is the type from {@link JavaTokenConstants}.
+     * @param tokenType is the type of the token.
      * @param position is the start position of the token.
      * @param length is the length of the token.
      */
-    private void addToken(int tokenType, long position, int length) {
+    private void addToken(JavaTokenType tokenType, long position, int length) {
         parser.add(tokenType, filename, map.getLineNumber(position), map.getColumnNumber(position), length);
     }
 
     /**
      * Convenience method that adds a specific token.
-     * @param tokenType is the type from {@link JavaTokenConstants}.
+     * @param tokenType is the type of the token.
      * @param start is the start position of the token.
      * @param end is the end position of the token for the calculation of the length.
      */
-    private void addToken(int tokenType, long start, long end) {
+    private void addToken(JavaTokenType tokenType, long start, long end) {
         parser.add(tokenType, filename, map.getLineNumber(start), map.getColumnNumber(start), (end - start));
     }
 
@@ -81,9 +81,9 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitBlock(BlockTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node) - 1;
-        addToken(JavaTokenConstants.J_INIT_BEGIN, start, 1);
+        addToken(JavaTokenType.J_INIT_BEGIN, start, 1);
         Object result = super.visitBlock(node, p);
-        addToken(JavaTokenConstants.J_INIT_END, end, 1);
+        addToken(JavaTokenType.J_INIT_END, end, 1);
         return result;
     }
 
@@ -93,27 +93,27 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
         long end = positions.getEndPosition(ast, node) - 1;
 
         if (node.getKind() == Tree.Kind.ENUM) {
-            addToken(JavaTokenConstants.J_ENUM_BEGIN, start, 4);
+            addToken(JavaTokenType.J_ENUM_BEGIN, start, 4);
         } else if (node.getKind() == Tree.Kind.INTERFACE) {
-            addToken(JavaTokenConstants.J_INTERFACE_BEGIN, start, 9);
+            addToken(JavaTokenType.J_INTERFACE_BEGIN, start, 9);
         } else if (node.getKind() == Tree.Kind.RECORD) {
-            addToken(JavaTokenConstants.J_RECORD_BEGIN, start, 1);
+            addToken(JavaTokenType.J_RECORD_BEGIN, start, 1);
         } else if (node.getKind() == Tree.Kind.ANNOTATION_TYPE) {
-            addToken(JavaTokenConstants.J_ANNO_T_BEGIN, start, 10);
+            addToken(JavaTokenType.J_ANNO_T_BEGIN, start, 10);
         } else if (node.getKind() == Tree.Kind.CLASS) {
-            addToken(JavaTokenConstants.J_CLASS_BEGIN, start, 5);
+            addToken(JavaTokenType.J_CLASS_BEGIN, start, 5);
         }
         Object result = super.visitClass(node, p);
         if (node.getKind() == Tree.Kind.ENUM) {
-            addToken(JavaTokenConstants.J_ENUM_END, end, 1);
+            addToken(JavaTokenType.J_ENUM_END, end, 1);
         } else if (node.getKind() == Tree.Kind.INTERFACE) {
-            addToken(JavaTokenConstants.J_INTERFACE_END, end, 1);
+            addToken(JavaTokenType.J_INTERFACE_END, end, 1);
         } else if (node.getKind() == Tree.Kind.RECORD) {
-            addToken(JavaTokenConstants.J_RECORD_END, end, 1);
+            addToken(JavaTokenType.J_RECORD_END, end, 1);
         } else if (node.getKind() == Tree.Kind.ANNOTATION_TYPE) {
-            addToken(JavaTokenConstants.J_ANNO_T_END, end, 1);
+            addToken(JavaTokenType.J_ANNO_T_END, end, 1);
         } else if (node.getKind() == Tree.Kind.CLASS) {
-            addToken(JavaTokenConstants.J_CLASS_END, end, 1);
+            addToken(JavaTokenType.J_CLASS_END, end, 1);
         }
         return result;
     }
@@ -121,14 +121,14 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     @Override
     public Object visitImport(ImportTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_IMPORT, start, 6);
+        addToken(JavaTokenType.J_IMPORT, start, 6);
         return super.visitImport(node, p);
     }
 
     @Override
     public Object visitPackage(PackageTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_PACKAGE, start, 7);
+        addToken(JavaTokenType.J_PACKAGE, start, 7);
         return super.visitPackage(node, p);
     }
 
@@ -136,9 +136,9 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitMethod(MethodTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node) - 1;
-        addToken(JavaTokenConstants.J_METHOD_BEGIN, start, node.getName().length());
+        addToken(JavaTokenType.J_METHOD_BEGIN, start, node.getName().length());
         Object result = super.visitMethod(node, p);
-        addToken(JavaTokenConstants.J_METHOD_END, end, 1);
+        addToken(JavaTokenType.J_METHOD_END, end, 1);
         return result;
     }
 
@@ -146,9 +146,9 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitSynchronized(SynchronizedTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node) - 1;
-        addToken(JavaTokenConstants.J_SYNC_BEGIN, start, 12);
+        addToken(JavaTokenType.J_SYNC_BEGIN, start, 12);
         Object result = super.visitSynchronized(node, p);
-        addToken(JavaTokenConstants.J_SYNC_END, end, 1);
+        addToken(JavaTokenType.J_SYNC_END, end, 1);
         return result;
     }
 
@@ -156,9 +156,9 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitDoWhileLoop(DoWhileLoopTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node) - 1;
-        addToken(JavaTokenConstants.J_DO_BEGIN, start, 2);
+        addToken(JavaTokenType.J_DO_BEGIN, start, 2);
         Object result = super.visitDoWhileLoop(node, p);
-        addToken(JavaTokenConstants.J_DO_END, end, 1);
+        addToken(JavaTokenType.J_DO_END, end, 1);
         return result;
     }
 
@@ -166,9 +166,9 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitWhileLoop(WhileLoopTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node) - 1;
-        addToken(JavaTokenConstants.J_WHILE_BEGIN, start, 5);
+        addToken(JavaTokenType.J_WHILE_BEGIN, start, 5);
         Object result = super.visitWhileLoop(node, p);
-        addToken(JavaTokenConstants.J_WHILE_END, end, 1);
+        addToken(JavaTokenType.J_WHILE_END, end, 1);
         return result;
     }
 
@@ -176,9 +176,9 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitForLoop(ForLoopTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node) - 1;
-        addToken(JavaTokenConstants.J_FOR_BEGIN, start, 3);
+        addToken(JavaTokenType.J_FOR_BEGIN, start, 3);
         Object result = super.visitForLoop(node, p);
-        addToken(JavaTokenConstants.J_FOR_END, end, 1);
+        addToken(JavaTokenType.J_FOR_END, end, 1);
         return result;
     }
 
@@ -186,9 +186,9 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitEnhancedForLoop(EnhancedForLoopTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node) - 1;
-        addToken(JavaTokenConstants.J_FOR_BEGIN, start, 3);
+        addToken(JavaTokenType.J_FOR_BEGIN, start, 3);
         Object result = super.visitEnhancedForLoop(node, p);
-        addToken(JavaTokenConstants.J_FOR_END, end, 1);
+        addToken(JavaTokenType.J_FOR_END, end, 1);
         return result;
     }
 
@@ -196,9 +196,9 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitSwitch(SwitchTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node) - 1;
-        addToken(JavaTokenConstants.J_SWITCH_BEGIN, start, 6);
+        addToken(JavaTokenType.J_SWITCH_BEGIN, start, 6);
         Object result = super.visitSwitch(node, p);
-        addToken(JavaTokenConstants.J_SWITCH_END, end, 1);
+        addToken(JavaTokenType.J_SWITCH_END, end, 1);
         return result;
     }
 
@@ -206,16 +206,16 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitSwitchExpression(SwitchExpressionTree node, Object parameterValue) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node) - 1;
-        addToken(JavaTokenConstants.J_SWITCH_BEGIN, start, 6);
+        addToken(JavaTokenType.J_SWITCH_BEGIN, start, 6);
         Object result = super.visitSwitchExpression(node, parameterValue);
-        addToken(JavaTokenConstants.J_SWITCH_END, end, 1);
+        addToken(JavaTokenType.J_SWITCH_END, end, 1);
         return result;
     }
 
     @Override
     public Object visitCase(CaseTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_CASE, start, 4);
+        addToken(JavaTokenType.J_CASE, start, 4);
         return super.visitCase(node, p);
     }
 
@@ -223,11 +223,11 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitTry(TryTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         if (node.getResources().isEmpty())
-            addToken(JavaTokenConstants.J_TRY_BEGIN, start, 3);
+            addToken(JavaTokenType.J_TRY_BEGIN, start, 3);
         else
-            addToken(JavaTokenConstants.J_TRY_WITH_RESOURCE, start, 3);
+            addToken(JavaTokenType.J_TRY_WITH_RESOURCE, start, 3);
         if (node.getFinallyBlock() != null)
-            addToken(JavaTokenConstants.J_FINALLY, start, 3);
+            addToken(JavaTokenType.J_FINALLY, start, 3);
         return super.visitTry(node, p);
     }
 
@@ -235,9 +235,9 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitCatch(CatchTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node) - 1;
-        addToken(JavaTokenConstants.J_CATCH_BEGIN, start, 5);
+        addToken(JavaTokenType.J_CATCH_BEGIN, start, 5);
         Object result = super.visitCatch(node, p);
-        addToken(JavaTokenConstants.J_CATCH_END, end, 1);
+        addToken(JavaTokenType.J_CATCH_END, end, 1);
         return result;
     }
 
@@ -245,43 +245,43 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitIf(IfTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node) - 1;
-        addToken(JavaTokenConstants.J_IF_BEGIN, start, 2);
+        addToken(JavaTokenType.J_IF_BEGIN, start, 2);
         node.getCondition().accept(this, p);
         node.getThenStatement().accept(this, p);
         if (node.getElseStatement() != null) {
             start = positions.getStartPosition(ast, node.getElseStatement());
-            addToken(JavaTokenConstants.J_ELSE, start, 4);
+            addToken(JavaTokenType.J_ELSE, start, 4);
             node.getElseStatement().accept(this, p);
         }
-        addToken(JavaTokenConstants.J_IF_END, end, 1);
+        addToken(JavaTokenType.J_IF_END, end, 1);
         return null;
     }
 
     @Override
     public Object visitBreak(BreakTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_BREAK, start, 5);
+        addToken(JavaTokenType.J_BREAK, start, 5);
         return super.visitBreak(node, p);
     }
 
     @Override
     public Object visitContinue(ContinueTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_CONTINUE, start, 8);
+        addToken(JavaTokenType.J_CONTINUE, start, 8);
         return super.visitContinue(node, p);
     }
 
     @Override
     public Object visitReturn(ReturnTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_RETURN, start, 6);
+        addToken(JavaTokenType.J_RETURN, start, 6);
         return super.visitReturn(node, p);
     }
 
     @Override
     public Object visitThrow(ThrowTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_THROW, start, 5);
+        addToken(JavaTokenType.J_THROW, start, 5);
         return super.visitThrow(node, p);
     }
 
@@ -289,9 +289,9 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitNewClass(NewClassTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         if (node.getTypeArguments().size() > 0) {
-            addToken(JavaTokenConstants.J_GENERIC, start, 3 + node.getIdentifier().toString().length());
+            addToken(JavaTokenType.J_GENERIC, start, 3 + node.getIdentifier().toString().length());
         }
-        addToken(JavaTokenConstants.J_NEWCLASS, start, 3);
+        addToken(JavaTokenType.J_NEWCLASS, start, 3);
         return super.visitNewClass(node, p);
     }
 
@@ -299,7 +299,7 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitTypeParameter(TypeParameterTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         // This is odd, but also done like this in Java17
-        addToken(JavaTokenConstants.J_GENERIC, start, 1);
+        addToken(JavaTokenType.J_GENERIC, start, 1);
         return super.visitTypeParameter(node, p);
     }
 
@@ -307,11 +307,11 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitNewArray(NewArrayTree node, Object arg1) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node) - 1;
-        addToken(JavaTokenConstants.J_NEWARRAY, start, 3);
+        addToken(JavaTokenType.J_NEWARRAY, start, 3);
         if (node.getInitializers() != null && !node.getInitializers().isEmpty()) {
             start = positions.getStartPosition(ast, node.getInitializers().get(0));
-            addToken(JavaTokenConstants.J_ARRAY_INIT_BEGIN, start, 1);
-            addToken(JavaTokenConstants.J_ARRAY_INIT_END, end, 1);
+            addToken(JavaTokenType.J_ARRAY_INIT_BEGIN, start, 1);
+            addToken(JavaTokenType.J_ARRAY_INIT_END, end, 1);
         }
         return super.visitNewArray(node, arg1);
     }
@@ -319,42 +319,42 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     @Override
     public Object visitAssignment(AssignmentTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_ASSIGN, start, 1);
+        addToken(JavaTokenType.J_ASSIGN, start, 1);
         return super.visitAssignment(node, p);
     }
 
     @Override
     public Object visitAssert(AssertTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_ASSERT, start, 6);
+        addToken(JavaTokenType.J_ASSERT, start, 6);
         return super.visitAssert(node, p);
     }
 
     @Override
     public Object visitVariable(VariableTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_VARDEF, start, node.toString().length());
+        addToken(JavaTokenType.J_VARDEF, start, node.toString().length());
         return super.visitVariable(node, p);
     }
 
     @Override
     public Object visitConditionalExpression(ConditionalExpressionTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_COND, start, 1);
+        addToken(JavaTokenType.J_COND, start, 1);
         return super.visitConditionalExpression(node, p);
     }
 
     @Override
     public Object visitMethodInvocation(MethodInvocationTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_APPLY, start, positions.getEndPosition(ast, node.getMethodSelect()) - start);
+        addToken(JavaTokenType.J_APPLY, start, positions.getEndPosition(ast, node.getMethodSelect()) - start);
         return super.visitMethodInvocation(node, p);
     }
 
     @Override
     public Object visitAnnotation(AnnotationTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_ANNO, start, 1);
+        addToken(JavaTokenType.J_ANNO, start, 1);
         return super.visitAnnotation(node, p);
     }
 
@@ -362,30 +362,30 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitModule(ModuleTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node) - 1;
-        addToken(JavaTokenConstants.J_MODULE_BEGIN, start, 6);
+        addToken(JavaTokenType.J_MODULE_BEGIN, start, 6);
         Object result = super.visitModule(node, p);
-        addToken(JavaTokenConstants.J_MODULE_END, end, 1);
+        addToken(JavaTokenType.J_MODULE_END, end, 1);
         return result;
     }
 
     @Override
     public Object visitRequires(RequiresTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_REQUIRES, start, 8);
+        addToken(JavaTokenType.J_REQUIRES, start, 8);
         return super.visitRequires(node, p);
     }
 
     @Override
     public Object visitProvides(ProvidesTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_PROVIDES, start, 8);
+        addToken(JavaTokenType.J_PROVIDES, start, 8);
         return super.visitProvides(node, p);
     }
 
     @Override
     public Object visitExports(ExportsTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
-        addToken(JavaTokenConstants.J_EXPORTS, start, 7);
+        addToken(JavaTokenType.J_EXPORTS, start, 7);
         return super.visitExports(node, p);
     }
 
@@ -399,7 +399,7 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitYield(YieldTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node);
-        addToken(JavaTokenConstants.J_YIELD, start, end);
+        addToken(JavaTokenType.J_YIELD, start, end);
         return super.visitYield(node, p);
     }
 
@@ -407,7 +407,7 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitDefaultCaseLabel(DefaultCaseLabelTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node);
-        addToken(JavaTokenConstants.J_DEFAULT, start, end);
+        addToken(JavaTokenType.J_DEFAULT, start, end);
         return super.visitDefaultCaseLabel(node, p);
     }
 }

languages/kotlin/src/main/java/de/jplag/kotlin/JPlagKotlinListener.java

@@ -1,6 +1,6 @@
 package de.jplag.kotlin;
 
-import static de.jplag.kotlin.KotlinTokenConstants.*;
+import static de.jplag.kotlin.KotlinTokenType.*;
 
 import java.util.Optional;
 
@@ -22,7 +22,7 @@ public class JPlagKotlinListener extends KotlinParserBaseListener {
      * @param tokenType the custom token type that occurred.
      * @param token the corresponding grammar's token
      */
-    private void transformToken(int tokenType, Token token) {
+    private void transformToken(KotlinTokenType tokenType, Token token) {
         parserAdapter.addToken(tokenType, token.getLine(), token.getCharPositionInLine() + 1, token.getText().length());
     }
 
@@ -33,7 +33,7 @@ public class JPlagKotlinListener extends KotlinParserBaseListener {
      * @param start the first Token of the context
      * @param end the last Token of the context
      */
-    private void transformToken(int tokenType, Token start, Token end) {
+    private void transformToken(KotlinTokenType tokenType, Token start, Token end) {
         parserAdapter.addToken(tokenType, start.getLine(), start.getCharPositionInLine() + 1, end.getStopIndex() - start.getStartIndex() + 1);
     }
 
@@ -356,7 +356,7 @@ public class JPlagKotlinListener extends KotlinParserBaseListener {
         if (tokenText.contains("@")) {
             tokenText = tokenText.substring(0, tokenText.indexOf("@"));
         }
-        Optional<Integer> type = switch (tokenText) {
+        Optional<KotlinTokenType> type = switch (tokenText) {
             case "throw" -> Optional.of(THROW);
             case "return" -> Optional.of(RETURN);
             case "continue" -> Optional.of(CONTINUE);

languages/kotlin/src/main/java/de/jplag/kotlin/KotlinParserAdapter.java

@@ -14,13 +14,10 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
 import de.jplag.Token;
-import de.jplag.TokenConstants;
 import de.jplag.kotlin.grammar.KotlinLexer;
 import de.jplag.kotlin.grammar.KotlinParser;
 
 public class KotlinParserAdapter extends AbstractParser {
-
-    public static final int NOT_SET = -1;
     private String currentFile;
     private List<Token> tokens;
 
@@ -43,7 +40,7 @@ public class KotlinParserAdapter extends AbstractParser {
             if (!parseFile(directory, file)) {
                 errors++;
             }
-            tokens.add(new KotlinToken(TokenConstants.FILE_END, file, NOT_SET, NOT_SET, NOT_SET));
+            tokens.add(Token.fileEnd(file));
         }
         return tokens;
     }
@@ -79,7 +76,7 @@ public class KotlinParserAdapter extends AbstractParser {
      * @param column the start column of the Token in the line
      * @param length the length of the Token
      */
-    /* package-private */ void addToken(int tokenType, int line, int column, int length) {
-        tokens.add(new KotlinToken(tokenType, currentFile, line, column, length));
+    /* package-private */ void addToken(KotlinTokenType tokenType, int line, int column, int length) {
+        tokens.add(new Token(tokenType, currentFile, line, column, length));
     }
 }

languages/kotlin/src/main/java/de/jplag/kotlin/KotlinToken.java

@@ -1,75 +0,0 @@
-package de.jplag.kotlin;
-
-import static de.jplag.kotlin.KotlinTokenConstants.*;
-
-import de.jplag.Token;
-
-public class KotlinToken extends Token {
-
-    public KotlinToken(int type, String file, int line, int column, int length) {
-        super(type, file, line, column, length);
-    }
-
-    @Override
-    public String type2string() {
-        return switch (type) {
-            case FILE_END -> "<EOF>";
-            case SEPARATOR_TOKEN -> "-----";
-            case PACKAGE -> "PACKAGE";
-            case IMPORT -> "IMPORT";
-            case CLASS_DECLARATION -> "CLASS";
-            case OBJECT_DECLARATION -> "OBJECT";
-            case COMPANION_DECLARATION -> "COMPANION";
-            case TYPE_PARAMETER -> "<T>";
-            case CONSTRUCTOR -> "CONSTRUCTOR";
-            case CLASS_BODY_BEGIN -> "CLASS{";
-            case CLASS_BODY_END -> "}CLASS";
-            case ENUM_CLASS_BODY_BEGIN -> "ENUM{";
-            case ENUM_CLASS_BODY_END -> "}ENUM";
-            case PROPERTY_DECLARATION -> "PROPERTY";
-            case INITIALIZER -> "INIT";
-            case INITIALIZER_BODY_START -> "INIT{";
-            case INITIALIZER_BODY_END -> "}INIT";
-            case FUNCTION -> "FUN";
-            case GETTER -> "GET";
-            case SETTER -> "SET";
-            case FUNCTION_PARAMETER -> "PARAM";
-            case FUNCTION_BODY_BEGIN -> "FUN{";
-            case FUNCTION_BODY_END -> "}FUN";
-            case FUNCTION_LITERAL_BEGIN -> "FUNC_LIT{";
-            case FUNCTION_LITERAL_END -> "}FUNC_LIT";
-            case FOR_EXPRESSION_BEGIN -> "FOR";
-            case FOR_EXPRESSION_END -> "}FOR";
-            case IF_EXPRESSION_START -> "IF";
-            case IF_EXPRESSION_END -> "}IF(-ELSE)";
-            case WHILE_EXPRESSION_START -> "WHILE";
-            case WHILE_EXPRESSION_END -> "}WHILE";
-            case DO_WHILE_EXPRESSION_START -> "DO";
-            case DO_WHILE_EXPRESSION_END -> "}DO-WHILE";
-            case TRY_EXPRESSION -> "TRY";
-            case TRY_BODY_START -> "TRY{";
-            case TRY_BODY_END -> "}TRY";
-            case CATCH -> "CATCH";
-            case CATCH_BODY_START -> "CATCH{";
-            case CATCH_BODY_END -> "}CATCH";
-            case FINALLY -> "FINALLY";
-            case FINALLY_BODY_START -> "FINALLY{";
-            case FINALLY_BODY_END -> "}FINALLY";
-            case WHEN_EXPRESSION_START -> "WHEN";
-            case WHEN_EXPRESSION_END -> "}WHEN";
-            case WHEN_CONDITION -> "COND";
-            case CONTROL_STRUCTURE_BODY_START -> "THEN{";
-            case CONTROL_STRUCTURE_BODY_END -> "}THEN";
-            case VARIABLE_DECLARATION -> "VARDECL";
-            case ENUM_ENTRY -> "ENUM_ENTRY";
-            case FUNCTION_INVOCATION -> "INVOC";
-            case CREATE_OBJECT -> "CONST";
-            case ASSIGNMENT -> "ASSIGN";
-            case THROW -> "THROW";
-            case RETURN -> "RETURN";
-            case CONTINUE -> "CONTINUE";
-            case BREAK -> "BREAK";
-            default -> "<UNKNOWN%d>".formatted(type);
-        };
-    }
-}

languages/kotlin/src/main/java/de/jplag/kotlin/KotlinTokenConstants.java

@@ -1,79 +0,0 @@
-package de.jplag.kotlin;
-
-import de.jplag.TokenConstants;
-
-public interface KotlinTokenConstants extends TokenConstants {
-
-    int PACKAGE = 2;
-    int IMPORT = 3;
-    int CLASS_DECLARATION = 4;
-    int OBJECT_DECLARATION = 5;
-    int COMPANION_DECLARATION = 6;
-
-    int TYPE_PARAMETER = 7;
-    int CONSTRUCTOR = 8;
-
-    int CLASS_BODY_BEGIN = 9;
-    int CLASS_BODY_END = 10;
-
-    int ENUM_CLASS_BODY_BEGIN = 11;
-    int ENUM_CLASS_BODY_END = 12;
-
-    int PROPERTY_DECLARATION = 13;
-
-    int INITIALIZER = 14;
-    int INITIALIZER_BODY_START = 15;
-    int INITIALIZER_BODY_END = 16;
-
-    int FUNCTION = 17;
-
-    int GETTER = 18;
-    int SETTER = 19;
-    int FUNCTION_PARAMETER = 20;
-    int FUNCTION_BODY_BEGIN = 21;
-    int FUNCTION_BODY_END = 22;
-
-    int FUNCTION_LITERAL_BEGIN = 23;
-    int FUNCTION_LITERAL_END = 24;
-
-    int FOR_EXPRESSION_BEGIN = 25;
-    int FOR_EXPRESSION_END = 26;
-
-    int IF_EXPRESSION_START = 27;
-    int IF_EXPRESSION_END = 28;
-
-    int WHILE_EXPRESSION_START = 29;
-    int WHILE_EXPRESSION_END = 30;
-
-    int DO_WHILE_EXPRESSION_START = 31;
-    int DO_WHILE_EXPRESSION_END = 32;
-
-    int TRY_EXPRESSION = 33;
-    int TRY_BODY_START = 34;
-    int TRY_BODY_END = 35;
-    int CATCH = 36;
-    int CATCH_BODY_START = 37;
-    int CATCH_BODY_END = 38;
-    int FINALLY = 39;
-    int FINALLY_BODY_START = 40;
-    int FINALLY_BODY_END = 41;
-    int WHEN_EXPRESSION_START = 42;
-    int WHEN_EXPRESSION_END = 43;
-    int WHEN_CONDITION = 44;
-
-    int CONTROL_STRUCTURE_BODY_START = 45;
-    int CONTROL_STRUCTURE_BODY_END = 46;
-
-    int VARIABLE_DECLARATION = 47;
-    int ENUM_ENTRY = 48;
-    int FUNCTION_INVOCATION = 49;
-    int CREATE_OBJECT = 50;
-    int ASSIGNMENT = 51;
-
-    int THROW = 52;
-    int RETURN = 53;
-    int CONTINUE = 54;
-    int BREAK = 55;
-
-    int NUMBER_DIFF_TOKENS = 56;
-}

languages/kotlin/src/main/java/de/jplag/kotlin/KotlinTokenType.java

@@ -0,0 +1,86 @@
+package de.jplag.kotlin;
+
+import de.jplag.TokenType;
+
+public enum KotlinTokenType implements TokenType {
+    PACKAGE("PACKAGE"),
+    IMPORT("IMPORT"),
+    CLASS_DECLARATION("CLASS"),
+    OBJECT_DECLARATION("OBJECT"),
+    COMPANION_DECLARATION("COMPANION"),
+
+    TYPE_PARAMETER("<T>"),
+    CONSTRUCTOR("CONSTRUCTOR"),
+
+    CLASS_BODY_BEGIN("CLASS{"),
+    CLASS_BODY_END("}CLASS"),
+
+    ENUM_CLASS_BODY_BEGIN("ENUM{"),
+    ENUM_CLASS_BODY_END("}ENUM"),
+
+    PROPERTY_DECLARATION("PROPERTY"),
+
+    INITIALIZER("INIT"),
+    INITIALIZER_BODY_START("INIT{"),
+    INITIALIZER_BODY_END("}INIT"),
+
+    FUNCTION("FUN"),
+
+    GETTER("GET"),
+    SETTER("SET"),
+    FUNCTION_PARAMETER("PARAM"),
+    FUNCTION_BODY_BEGIN("FUN{"),
+    FUNCTION_BODY_END("}FUN"),
+
+    FUNCTION_LITERAL_BEGIN("FUNC_LIT{"),
+    FUNCTION_LITERAL_END("}FUNC_LIT"),
+
+    FOR_EXPRESSION_BEGIN("FOR"),
+    FOR_EXPRESSION_END("}FOR"),
+
+    IF_EXPRESSION_START("IF"),
+    IF_EXPRESSION_END("}IF(-ELSE)"),
+
+    WHILE_EXPRESSION_START("WHILE"),
+    WHILE_EXPRESSION_END("}WHILE"),
+
+    DO_WHILE_EXPRESSION_START("DO"),
+    DO_WHILE_EXPRESSION_END("}DO-WHILE"),
+
+    TRY_EXPRESSION("TRY"),
+    TRY_BODY_START("TRY{"),
+    TRY_BODY_END("}TRY"),
+    CATCH("CATCH"),
+    CATCH_BODY_START("CATCH{"),
+    CATCH_BODY_END("}CATCH"),
+    FINALLY("FINALLY"),
+    FINALLY_BODY_START("FINALLY{"),
+    FINALLY_BODY_END("}FINALLY"),
+    WHEN_EXPRESSION_START("WHEN"),
+    WHEN_EXPRESSION_END("}WHEN"),
+    WHEN_CONDITION("COND"),
+
+    CONTROL_STRUCTURE_BODY_START("THEN{"),
+    CONTROL_STRUCTURE_BODY_END("}THEN"),
+
+    VARIABLE_DECLARATION("VARDECL"),
+    ENUM_ENTRY("ENUM_ENTRY"),
+    FUNCTION_INVOCATION("INVOC"),
+    CREATE_OBJECT("CONST"),
+    ASSIGNMENT("ASSIGN"),
+
+    THROW("THROW"),
+    RETURN("RETURN"),
+    CONTINUE("CONTINUE"),
+    BREAK("BREAK");
+
+    private final String description;
+
+    public String getDescription() {
+        return description;
+    }
+
+    KotlinTokenType(String description) {
+        this.description = description;
+    }
+}

languages/kotlin/src/test/java/de/jplag/kotlin/KotlinFrontendTest.java

@@ -1,12 +1,14 @@
 package de.jplag.kotlin;
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -16,8 +18,8 @@ import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import de.jplag.SharedTokenType;
 import de.jplag.Token;
-import de.jplag.TokenConstants;
 import de.jplag.TokenPrinter;
 
 class KotlinFrontendTest {
@@ -41,8 +43,6 @@ class KotlinFrontendTest {
      * Regular expression that describes lines containing the end of a multiline comment and no more code after that.
      */
     private static final String DELIMITED_COMMENT_END = ".*\\*/\\s*$";
-    private static final String NOT_SET_STRING = "";
-    private static final int NOT_SET = -1;
 
     private final Logger logger = LoggerFactory.getLogger("Kotlin frontend test");
     private final String[] testFiles = new String[] {COMPLETE_TEST_FILE, "Game.kt"};
@@ -69,22 +69,6 @@ class KotlinFrontendTest {
         }
     }
 
-    /**
-     * Confirms that every type of KotlinToken has a Sting representation associated to it.
-     */
-    @Test
-    void testTokenToString() {
-        var missingTokens = IntStream.range(0, KotlinTokenConstants.NUMBER_DIFF_TOKENS)
-                .mapToObj(type -> new KotlinToken(type, NOT_SET_STRING, NOT_SET, NOT_SET, NOT_SET))
-                .filter(token -> token.type2string().contains("UNKNOWN")).toList();
-
-        if (!missingTokens.isEmpty()) {
-            var typeList = missingTokens.stream().map(Token::getType).map(Object::toString).collect(Collectors.joining(", "));
-            fail("Found token types with no string representation: %s".formatted(typeList));
-        }
-
-    }
-
     /**
      * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
      * @param fileName a code sample file name
@@ -152,17 +136,13 @@ class KotlinFrontendTest {
      * @param fileName The file name of the complete code example
      */
     private void testTokenCoverage(List<Token> tokens, String fileName) {
-        var foundTokens = tokens.stream().parallel().mapToInt(Token::getType).sorted().distinct().toArray();
-        // Exclude SEPARATOR_TOKEN, as it does not occur
-        var allTokens = IntStream.range(0, KotlinTokenConstants.NUMBER_DIFF_TOKENS).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).toArray();
-
-        if (allTokens.length > foundTokens.length) {
-            var diffLine = IntStream.range(0, allTokens.length)
-                    .dropWhile(lineIdx -> lineIdx < foundTokens.length && allTokens[lineIdx] == foundTokens[lineIdx]).findFirst();
-            diffLine.ifPresent(lineIdx -> fail("Token type %s was not found in the complete code example '%s'."
-                    .formatted(new KotlinToken(allTokens[lineIdx], fileName, -1, -1, -1).type2string(), fileName)));
-        }
-        assertArrayEquals(allTokens, foundTokens);
+        var annotatedTokens = tokens.stream().map(Token::getType).collect(Collectors.toSet());
+        assertTrue(annotatedTokens.contains(SharedTokenType.FILE_END));
+        var annotatedKotlinTokens = annotatedTokens.stream().filter(KotlinTokenType.class::isInstance).collect(Collectors.toSet());
+        var allKotlinTokens = KotlinTokenType.values();
+        var missingKotlinTokens = Arrays.stream(allKotlinTokens).filter(token -> !annotatedKotlinTokens.contains(token)).toList();
+        assertTrue(missingKotlinTokens.isEmpty(), "The following kotlin tokens are missing in the code example '%s':\n".formatted(fileName)
+                + String.join("\n", missingKotlinTokens.stream().map(KotlinTokenType::getDescription).toList()));
     }
 
 }

languages/python-3/src/main/java/de/jplag/python3/JplagPython3Listener.java

@@ -1,11 +1,13 @@
 package de.jplag.python3;
 
+import static de.jplag.python3.Python3TokenType.*;
+
 import org.antlr.v4.runtime.tree.TerminalNode;
 
 import de.jplag.python3.grammar.Python3BaseListener;
 import de.jplag.python3.grammar.Python3Parser;
 
-public class JplagPython3Listener extends Python3BaseListener implements Python3TokenConstants {
+public class JplagPython3Listener extends Python3BaseListener {
 
     private final Parser parser;
 

languages/python-3/src/main/java/de/jplag/python3/Parser.java

@@ -10,19 +10,19 @@ import java.util.List;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
-import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenConstants;
+import de.jplag.Token;
+import de.jplag.TokenType;
 import de.jplag.python3.grammar.Python3Lexer;
 import de.jplag.python3.grammar.Python3Parser;
 import de.jplag.python3.grammar.Python3Parser.File_inputContext;
 
 public class Parser extends AbstractParser {
 
-    private List<de.jplag.Token> tokens;
+    private List<Token> tokens;
     private String currentFile;
 
     /**
@@ -32,7 +32,7 @@ public class Parser extends AbstractParser {
         super();
     }
 
-    public List<de.jplag.Token> parse(File directory, String[] files) {
+    public List<Token> parse(File directory, String[] files) {
         tokens = new ArrayList<>();
         errors = 0;
         for (String file : files) {
@@ -40,7 +40,7 @@ public class Parser extends AbstractParser {
             if (!parseFile(directory, file)) {
                 errors++;
             }
-            tokens.add(new Python3Token(TokenConstants.FILE_END, file, -1, -1, -1));
+            tokens.add(Token.fileEnd(file));
         }
         return tokens;
     }
@@ -78,13 +78,12 @@ public class Parser extends AbstractParser {
         return true;
     }
 
-    public void add(int type, Token token) {
-        tokens.add(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(), token.getCharPositionInLine() + 1,
+    public void add(TokenType type, org.antlr.v4.runtime.Token token) {
+        tokens.add(new Token(type, (currentFile == null ? "null" : currentFile), token.getLine(), token.getCharPositionInLine() + 1,
                 token.getText().length()));
     }
 
-    public void addEnd(int type, Token token) {
-        tokens.add(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(),
-                tokens.get(tokens.size() - 1).getColumn() + 1, 0));
+    public void addEnd(TokenType type, org.antlr.v4.runtime.Token token) {
+        tokens.add(new Token(type, (currentFile == null ? "null" : currentFile), token.getLine(), tokens.get(tokens.size() - 1).getColumn() + 1, 0));
     }
 }

languages/python-3/src/main/java/de/jplag/python3/Python3Token.java

@@ -1,88 +0,0 @@
-package de.jplag.python3;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import de.jplag.Token;
-
-public class Python3Token extends Token implements Python3TokenConstants {
-
-    private static final Logger logger = LoggerFactory.getLogger(Python3Token.class);
-
-    public Python3Token(int type, String file, int line, int column, int length) {
-        super(type, file, line, column, length);
-    }
-
-    protected String type2string() {
-        switch (type) {
-            case Python3TokenConstants.FILE_END:
-                return "EOF";
-            case Python3TokenConstants.SEPARATOR_TOKEN:
-                return "METHOD_SEPARATOR";
-
-            case IMPORT:
-                return "IMPORT  ";
-            case CLASS_BEGIN:
-                return "CLASS{  ";
-            case CLASS_END:
-                return "}CLASS  ";
-            case METHOD_BEGIN:
-                return "METHOD{ ";
-            case METHOD_END:
-                return "}METHOD ";
-            case ASSIGN:
-                return "ASSIGN  ";
-            case WHILE_BEGIN:
-                return "WHILE{  ";
-            case WHILE_END:
-                return "}WHILE  ";
-            case FOR_BEGIN:
-                return "FOR{    ";
-            case FOR_END:
-                return "}FOR    ";
-            case TRY_BEGIN:
-                return "TRY{    ";
-            case EXCEPT_BEGIN:
-                return "CATCH{  ";
-            case EXCEPT_END:
-                return "}CATCH  ";
-            case FINALLY:
-                return "FINALLY ";
-            case IF_BEGIN:
-                return "IF{     ";
-            case IF_END:
-                return "}IF     ";
-            case APPLY:
-                return "APPLY   ";
-            case BREAK:
-                return "BREAK   ";
-            case CONTINUE:
-                return "CONTINUE";
-            case RETURN:
-                return "RETURN  ";
-            case RAISE:
-                return "RAISE   ";
-            case DEC_BEGIN:
-                return "DECOR{  ";
-            case DEC_END:
-                return "}DECOR  ";
-            case LAMBDA:
-                return "LAMBDA  ";
-            case ARRAY:
-                return "ARRAY   ";
-            case ASSERT:
-                return "ASSERT  ";
-            case YIELD:
-                return "YIELD   ";
-            case DEL:
-                return "DEL     ";
-            case WITH_BEGIN:
-                return "WITH{   ";
-            case WITH_END:
-                return "}WITH   ";
-            default:
-                logger.error("*UNKNOWN: " + type);
-                return "*UNKNOWN" + type;
-        }
-    }
-}

languages/python-3/src/main/java/de/jplag/python3/Python3TokenConstants.java

@@ -1,38 +0,0 @@
-package de.jplag.python3;
-
-import de.jplag.TokenConstants;
-
-public interface Python3TokenConstants extends TokenConstants {
-    int IMPORT = 2;
-    int CLASS_BEGIN = 3;
-    int CLASS_END = 4;
-    int METHOD_BEGIN = 5;
-    int METHOD_END = 6;
-    int ASSIGN = 7;
-    int WHILE_BEGIN = 8;
-    int WHILE_END = 9;
-    int FOR_BEGIN = 10;
-    int FOR_END = 11;
-    int TRY_BEGIN = 12;
-    int EXCEPT_BEGIN = 13;
-    int EXCEPT_END = 14;
-    int FINALLY = 15;
-    int IF_BEGIN = 16;
-    int IF_END = 18;
-    int APPLY = 19;
-    int BREAK = 20;
-    int CONTINUE = 21;
-    int RETURN = 22;
-    int RAISE = 23;
-    int DEC_BEGIN = 24;
-    int DEC_END = 25;
-    int LAMBDA = 26;
-    int ARRAY = 27;
-    int ASSERT = 28;
-    int YIELD = 29;
-    int DEL = 30;
-    int WITH_BEGIN = 31;
-    int WITH_END = 32;
-
-    int NUM_DIFF_TOKENS = 33;
-}

languages/python-3/src/main/java/de/jplag/python3/Python3TokenType.java

@@ -0,0 +1,46 @@
+package de.jplag.python3;
+
+import de.jplag.TokenType;
+
+public enum Python3TokenType implements TokenType {
+    IMPORT("IMPORT"),
+    CLASS_BEGIN("CLASS{"),
+    CLASS_END("}CLASS"),
+    METHOD_BEGIN("METHOD{"),
+    METHOD_END("}METHOD"),
+    ASSIGN("ASSIGN"),
+    WHILE_BEGIN("WHILE{"),
+    WHILE_END("}WHILE"),
+    FOR_BEGIN("FOR{"),
+    FOR_END("}FOR"),
+    TRY_BEGIN("TRY{"),
+    EXCEPT_BEGIN("CATCH{"),
+    EXCEPT_END("}CATCH"),
+    FINALLY("FINALLY"),
+    IF_BEGIN("IF{"),
+    IF_END("}IF"),
+    APPLY("APPLY"),
+    BREAK("BREAK"),
+    CONTINUE("CONTINUE"),
+    RETURN("RETURN"),
+    RAISE("RAISE"),
+    DEC_BEGIN("DECOR{"),
+    DEC_END("}DECOR"),
+    LAMBDA("LAMBDA"),
+    ARRAY("ARRAY"),
+    ASSERT("ASSERT"),
+    YIELD("YIELD"),
+    DEL("DEL"),
+    WITH_BEGIN("WITH}"),
+    WITH_END("}WITH");
+
+    private final String description;
+
+    public String getDescription() {
+        return this.description;
+    }
+
+    Python3TokenType(String description) {
+        this.description = description;
+    }
+}

languages/rlang/src/main/java/de/jplag/rlang/JplagRListener.java

@@ -1,5 +1,7 @@
 package de.jplag.rlang;
 
+import static de.jplag.rlang.RTokenType.*;
+
 import org.antlr.v4.runtime.Token;
 
 import de.jplag.rlang.grammar.*;
@@ -10,7 +12,7 @@ import de.jplag.rlang.grammar.*;
  * <a href="https://github.com/CodeGra-de/jplag/tree/master/jplag.frontend.R">JPlag fork</a>.
  * @author Robin Maisch
  */
-public class JplagRListener extends RCombinedBaseListener implements RTokenConstants {
+public class JplagRListener extends RCombinedBaseListener {
 
     private final RParserAdapter parserAdapter;
 
@@ -27,11 +29,11 @@ public class JplagRListener extends RCombinedBaseListener implements RTokenConst
      * @param targetType the type of the JPlag token to be created.
      * @param token the ANTLR token.
      */
-    private void transformToken(int targetType, Token token) {
+    private void transformToken(RTokenType targetType, Token token) {
         parserAdapter.addToken(targetType, token.getLine(), token.getCharPositionInLine() + 1, token.getText().length());
     }
 
-    private void transformToken(int targetType, Token start, Token end) {
+    private void transformToken(RTokenType targetType, Token start, Token end) {
         parserAdapter.addToken(targetType, start.getLine(), start.getCharPositionInLine() + 1, end.getStopIndex() - start.getStartIndex() + 1);
     }
 

languages/rlang/src/main/java/de/jplag/rlang/RParserAdapter.java

@@ -14,6 +14,7 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
 import de.jplag.Token;
+import de.jplag.TokenType;
 import de.jplag.rlang.grammar.RFilter;
 import de.jplag.rlang.grammar.RLexer;
 import de.jplag.rlang.grammar.RParser;
@@ -22,7 +23,7 @@ import de.jplag.rlang.grammar.RParser;
  * This class sets up the lexer and parser generated by ANTLR4, feeds the submissions through them and passes the
  * selected tokens on to the main program.
  */
-public class RParserAdapter extends AbstractParser implements RTokenConstants {
+public class RParserAdapter extends AbstractParser {
 
     private String currentFile;
     private List<Token> tokens;
@@ -47,7 +48,7 @@ public class RParserAdapter extends AbstractParser implements RTokenConstants {
             if (!parseFile(directory, fileName)) {
                 errors++;
             }
-            tokens.add(new RToken(FILE_END, fileName, -1, -1, -1));
+            tokens.add(Token.fileEnd(fileName));
         }
         return tokens;
     }
@@ -90,8 +91,8 @@ public class RParserAdapter extends AbstractParser implements RTokenConstants {
      * @param start the start column of the Token in the line
      * @param length the length of the Token
      */
-    /* package-private */ void addToken(int type, int line, int start, int length) {
-        tokens.add(new RToken(type, currentFile, line, start, length));
+    /* package-private */ void addToken(TokenType type, int line, int start, int length) {
+        tokens.add(new Token(type, currentFile, line, start, length));
 
     }
 }

languages/rlang/src/main/java/de/jplag/rlang/RToken.java

@@ -1,46 +0,0 @@
-package de.jplag.rlang;
-
-/**
- * This class represents the occurrence of an R Token in the source code. Based on an R frontend for JPlag v2.15 by Olmo
- * Kramer, see their <a href="https://github.com/CodeGra-de/jplag/tree/master/jplag.frontend.R">JPlag fork</a>.
- */
-public class RToken extends de.jplag.Token implements RTokenConstants {
-
-    public RToken(int type, String file, int line, int column, int length) {
-        super(type, file, line, column, length);
-    }
-
-    @Override
-    public String type2string() {
-        return switch (this.type) {
-            case FILE_END -> "<EOF>";
-            case SEPARATOR_TOKEN -> "METHOD_SEPARATOR";
-            case BEGIN_FUNCTION -> "FUNCTION{";
-            case END_FUNCTION -> "}FUNCTION";
-            case FUNCTION_CALL -> "FUNCTION()";
-            case NUMBER -> "NUMBER";
-            case STRING -> "STRING";
-            case BOOL -> "BOOL";
-            case ASSIGN -> "ASSIGN";
-            case ASSIGN_FUNC -> "ASSIGN_FUNC";
-            case ASSIGN_LIST -> "ASSIGN_LIST";
-            case HELP -> "HELP";
-            case INDEX -> "INDEX";
-            case PACKAGE -> "PACKAGE";
-            case IF_BEGIN -> "IF{";
-            case IF_END -> "}IF-ELSE";
-            case FOR_BEGIN -> "FOR{";
-            case FOR_END -> "}FOR";
-            case WHILE_BEGIN -> "WHILE{";
-            case WHILE_END -> "}WHILE";
-            case REPEAT_BEGIN -> "REPEAT{";
-            case REPEAT_END -> "}REPEAT";
-            case NEXT -> "NEXT";
-            case BREAK -> "BREAK";
-            case COMPOUND_BEGIN -> "COMPOUND{";
-            case COMPOUND_END -> "}COMPOUND";
-            default -> "<UNKNOWN%d>".formatted(type);
-        };
-    }
-
-}

languages/rlang/src/main/java/de/jplag/rlang/RTokenConstants.java

@@ -1,39 +0,0 @@
-package de.jplag.rlang;
-
-import de.jplag.TokenConstants;
-
-/**
- * Tokens in R that are deemed important when comparing submissions for plagiarisms. Based on an R frontend for JPlag
- * v2.15 by Olmo Kramer, see their <a href="https://github.com/CodeGra-de/jplag/tree/master/jplag.frontend.R">JPlag
- * fork</a>.
- * @author Robin Maisch
- */
-public interface RTokenConstants extends TokenConstants {
-
-    int BEGIN_FUNCTION = 2;
-    int END_FUNCTION = 3;
-    int FUNCTION_CALL = 4;
-    int NUMBER = 5;
-    int STRING = 6;
-    int BOOL = 7;
-    int ASSIGN = 8;
-    int ASSIGN_FUNC = 9;
-    int ASSIGN_LIST = 10;
-    int HELP = 11;
-    int INDEX = 12;
-    int PACKAGE = 13;
-    int IF_BEGIN = 14;
-    int IF_END = 15;
-    int FOR_BEGIN = 16;
-    int FOR_END = 17;
-    int WHILE_BEGIN = 18;
-    int WHILE_END = 19;
-    int REPEAT_BEGIN = 20;
-    int REPEAT_END = 21;
-    int NEXT = 22;
-    int BREAK = 23;
-    int COMPOUND_BEGIN = 24;
-    int COMPOUND_END = 25;
-
-    int NUM_DIFF_TOKENS = 26;
-}

languages/rlang/src/main/java/de/jplag/rlang/RTokenType.java

@@ -0,0 +1,45 @@
+package de.jplag.rlang;
+
+import de.jplag.TokenType;
+
+/**
+ * Tokens in R that are deemed important when comparing submissions for plagiarisms. Based on an R frontend for JPlag
+ * v2.15 by Olmo Kramer, see their <a href="https://github.com/CodeGra-de/jplag/tree/master/jplag.frontend.R">JPlag
+ * fork</a>.
+ */
+public enum RTokenType implements TokenType {
+    BEGIN_FUNCTION("FUNCTION{"),
+    END_FUNCTION("}FUNCTION"),
+    FUNCTION_CALL("FUNCTION()"),
+    NUMBER("NUMBER"),
+    STRING("STRING"),
+    BOOL("BOOL"),
+    ASSIGN("ASSIGN"),
+    ASSIGN_FUNC("ASSIGN_FUNC"),
+    ASSIGN_LIST("ASSIGN_LIST"),
+    HELP("HELP"),
+    INDEX("INDEX"),
+    PACKAGE("PACKAGE"),
+    IF_BEGIN("IF{"),
+    IF_END("}IF-ELSE"),
+    FOR_BEGIN("FOR{"),
+    FOR_END("}FOR"),
+    WHILE_BEGIN("WHILE{"),
+    WHILE_END("}WHILE"),
+    REPEAT_BEGIN("REPEAT{"),
+    REPEAT_END("}REPEAT"),
+    NEXT("NEXT"),
+    BREAK("BREAK"),
+    COMPOUND_BEGIN("COMPOUND{"),
+    COMPOUND_END("}COMPOUND");
+
+    private final String description;
+
+    public String getDescription() {
+        return this.description;
+    }
+
+    RTokenType(String description) {
+        this.description = description;
+    }
+}

languages/rlang/src/test/java/de/jplag/rlang/RFrontendTest.java

@@ -1,13 +1,15 @@
 package de.jplag.rlang;
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
-import java.io.File;
-import java.io.IOException;
+import java.io.*;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import org.junit.jupiter.api.BeforeEach;
@@ -15,8 +17,8 @@ import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import de.jplag.SharedTokenType;
 import de.jplag.Token;
-import de.jplag.TokenConstants;
 import de.jplag.TokenPrinter;
 
 class RFrontendTest {
@@ -30,7 +32,6 @@ class RFrontendTest {
      * Test source file that is supposed to produce a complete set of tokens, i.e. all types of tokens.
      */
     private static final String COMPLETE_TEST_FILE = "Complete.R";
-    public static final int NOT_SET = -1;
 
     private final Logger logger = LoggerFactory.getLogger("R frontend test");
     private final String[] testFiles = new String[] {"Game.R", COMPLETE_TEST_FILE};
@@ -91,17 +92,13 @@ class RFrontendTest {
      * @param fileName The file name of the complete code example
      */
     private void testTokenCoverage(List<Token> tokens, String fileName) {
-        var foundTokens = tokens.stream().parallel().mapToInt(Token::getType).sorted().distinct().toArray();
-        // Exclude SEPARATOR_TOKEN, as it does not occur
-        var allTokens = IntStream.range(0, RTokenConstants.NUM_DIFF_TOKENS).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).toArray();
-
-        if (allTokens.length > foundTokens.length) {
-            var diffLine = IntStream.range(0, allTokens.length)
-                    .dropWhile(lineIndex -> lineIndex < foundTokens.length && allTokens[lineIndex] == foundTokens[lineIndex]).findFirst();
-            diffLine.ifPresent(lineIdx -> fail("Token type %s was not found in the complete code example '%s'."
-                    .formatted(new RToken(allTokens[lineIdx], fileName, NOT_SET, NOT_SET, NOT_SET).type2string(), fileName)));
-        }
-        assertArrayEquals(allTokens, foundTokens);
+        var annotatedTokens = tokens.stream().map(Token::getType).collect(Collectors.toSet());
+        assertTrue(annotatedTokens.contains(SharedTokenType.FILE_END));
+        var annotatedRTokens = annotatedTokens.stream().filter(RTokenType.class::isInstance).collect(Collectors.toSet());
+        var allRTokens = RTokenType.values();
+        var missingRTokens = Arrays.stream(allRTokens).filter(token -> !annotatedRTokens.contains(token)).toList();
+        assertTrue(missingRTokens.isEmpty(), "The following R tokens are missing in the code example '%s':\n".formatted(fileName)
+                + String.join("\n", missingRTokens.stream().map(RTokenType::getDescription).toList()));
     }
 
     private static String getNoCodeLineExpression() {

languages/rust/src/main/java/de/jplag/rust/JPlagRustListener.java

@@ -1,9 +1,9 @@
 package de.jplag.rust;
 
-import static de.jplag.rust.ParserState.Context.NONE;
-import static de.jplag.rust.RustTokenConstants.*;
+import static de.jplag.rust.RustTokenType.*;
 
 import java.util.Objects;
+import java.util.Optional;
 
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.RuleContext;
@@ -24,11 +24,15 @@ public class JPlagRustListener extends RustParserBaseListener implements ParseTr
         state.enter(RustContext.FILE);
     }
 
-    private void transformToken(int targetType, Token token) {
+    private void transformToken(Optional<RustTokenType> targetType, Token token) {
+        targetType.ifPresent(type -> transformToken(type, token));
+    }
+
+    private void transformToken(RustTokenType targetType, Token token) {
         parserAdapter.addToken(targetType, token.getLine(), token.getCharPositionInLine() + 1, token.getText().length());
     }
 
-    private void transformToken(int targetType, Token start, Token end) {
+    private void transformToken(RustTokenType targetType, Token start, Token end) {
         parserAdapter.addToken(targetType, start.getLine(), start.getCharPositionInLine() + 1, end.getStopIndex() - start.getStartIndex() + 1);
     }
 
@@ -97,7 +101,7 @@ public class JPlagRustListener extends RustParserBaseListener implements ParseTr
 
     @Override
     public void enterStructExpression(RustParser.StructExpressionContext context) {
-        transformToken(RustTokenConstants.STRUCT_INITIALISATION, context.getStart());
+        transformToken(STRUCT_INITIALISATION, context.getStart());
         state.enter(RustContext.STRUCT_INITIALISATION);
         super.enterStructExpression(context);
     }
@@ -353,7 +357,7 @@ public class JPlagRustListener extends RustParserBaseListener implements ParseTr
 
     @Override
     public void enterStaticItem(RustParser.StaticItemContext context) {
-        int tokenType = context.getParent() instanceof RustParser.ExternalItemContext ? STATIC_ITEM : VARIABLE_DECLARATION;
+        RustTokenType tokenType = context.getParent() instanceof RustParser.ExternalItemContext ? STATIC_ITEM : VARIABLE_DECLARATION;
         transformToken(tokenType, context.getStart());
         super.enterStaticItem(context);
     }
@@ -661,10 +665,8 @@ public class JPlagRustListener extends RustParserBaseListener implements ParseTr
                 }
             }
             case "{" -> {
-                int startType = stateContext.getStartType();
-                if (startType != NONE) {
-                    transformToken(startType, token);
-                }
+                Optional<RustTokenType> startType = stateContext.getStartType();
+                startType.ifPresent(type -> transformToken(type, token));
                 switch (stateContext) {
                     case MACRO_RULE_BODY, MACRO_INVOCATION_BODY, MACRO_INNER -> state.enter(RustContext.MACRO_INNER);
                     default -> {
@@ -674,10 +676,8 @@ public class JPlagRustListener extends RustParserBaseListener implements ParseTr
 
             }
             case "}" -> {
-                int endType = stateContext.getEndType();
-                if (endType != NONE) {
-                    transformToken(endType, token);
-                }
+                Optional<RustTokenType> endType = stateContext.getEndType();
+                endType.ifPresent(type -> transformToken(type, token));
 
                 if (stateContext == RustContext.MACRO_INNER) {
                     // maybe this is the end of a macro invocation/definition
@@ -809,7 +809,7 @@ public class JPlagRustListener extends RustParserBaseListener implements ParseTr
         /**
          * This is used to make sure that the stack is not empty -> getCurrent() != null
          **/
-        FILE(NONE, NONE),
+        FILE(),
 
         /**
          * These contexts are used to assign the correct tokens to '{' and '}' terminals.
@@ -838,54 +838,59 @@ public class JPlagRustListener extends RustParserBaseListener implements ParseTr
         /**
          * This is to avoid the empty type `()` being parsed as an empty tuple etc.
          **/
-        TYPE(NONE, NONE),
+        TYPE(),
 
         /**
          * These are to identify expressions as elements of tuples.
          */
-        TUPLE_STRUCT_PATTERN(NONE, NONE),
-        TUPLE_PATTERN(NONE, NONE),
+        TUPLE_STRUCT_PATTERN(),
+        TUPLE_PATTERN(),
 
         /**
          * This is used so that cascades of tuples like '((((1),2),(3)))' generate only as many tokens as necessary.
          */
-        REDUNDANT_TUPLE(NONE, NONE),
+        REDUNDANT_TUPLE(),
 
         /**
          * This is used to be able to correctly assign MACRO_INVOCATION_BODY_END to a '}' symbol.
          */
-        MACRO_INNER(NONE, NONE),
+        MACRO_INNER(),
 
         /**
          * In this context, leaves are USE_ITEMS.
          */
-        USE_TREE(NONE, NONE),
+        USE_TREE(),
 
         /**
          * In this context, '(' should be assigned an APPLY token.
          */
-        CALL(NONE, NONE),
+        CALL(),
 
         /**
          * This context should behave like a function call: No tokens for parentheses.
          */
-        STRUCT_INITIALISATION(NONE, NONE);
+        STRUCT_INITIALISATION();
+
+        private final Optional<RustTokenType> startType;
+        private final Optional<RustTokenType> endType;
 
-        private final int startType;
-        private final int endType;
+        RustContext() {
+            this.startType = Optional.empty();
+            this.endType = Optional.empty();
+        }
 
-        RustContext(int startType, int endType) {
-            this.startType = startType;
-            this.endType = endType;
+        RustContext(RustTokenType startType, RustTokenType endType) {
+            this.startType = Optional.of(startType);
+            this.endType = Optional.of(endType);
         }
 
         @Override
-        public int getStartType() {
+        public Optional<RustTokenType> getStartType() {
             return startType;
         }
 
         @Override
-        public int getEndType() {
+        public Optional<RustTokenType> getEndType() {
             return endType;
         }
     }

languages/rust/src/main/java/de/jplag/rust/ParserState.java

@@ -3,6 +3,7 @@ package de.jplag.rust;
 import java.util.Arrays;
 import java.util.Deque;
 import java.util.LinkedList;
+import java.util.Optional;
 
 /**
  * A ParserState is a representation for the state of a parser, consisting of a stack of Contexts.
@@ -56,25 +57,20 @@ public class ParserState<C extends ParserState.Context> {
 
     /**
      * A Context is a grammatical situation, e.g. a class body, or a while statement. Each Context should have a startType
-     * and an endType, designating the start and the end of the context as a TokenConstant.
+     * and an endType, designating the start and the end of the context as a token type.
      */
     protected interface Context {
 
         /**
-         * Used as start or end type to indicate that no token shall be added for this context.
-         */
-        int NONE = -1;
-
-        /**
-         * Returns the TokenConstant that marks the start of the Context.
+         * Returns the token type that marks the start of the Context.
          * @return the start type
          */
-        int getStartType();
+        Optional<RustTokenType> getStartType();
 
         /**
-         * The TokenConstant that marks the end of the Context.
+         * Returns the token type that marks the end of the Context.
          * @return the end type
          */
-        int getEndType();
+        Optional<RustTokenType> getEndType();
     }
 }

languages/rust/src/main/java/de/jplag/rust/RustParserAdapter.java

@@ -14,13 +14,11 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
 import de.jplag.Token;
-import de.jplag.TokenConstants;
 import de.jplag.rust.grammar.RustLexer;
 import de.jplag.rust.grammar.RustParser;
 
 public class RustParserAdapter extends AbstractParser {
 
-    private static final int NOT_SET = -1;
     private String currentFile;
     private List<Token> tokens;
 
@@ -37,7 +35,7 @@ public class RustParserAdapter extends AbstractParser {
             if (!parseFile(directory, fileName)) {
                 errors++;
             }
-            tokens.add(new RustToken(TokenConstants.FILE_END, fileName, NOT_SET, NOT_SET, NOT_SET));
+            tokens.add(Token.fileEnd(fileName));
         }
         return tokens;
     }
@@ -76,8 +74,8 @@ public class RustParserAdapter extends AbstractParser {
      * @param start the start column of the Token in the line
      * @param length the length of the Token
      */
-    /* package-private */ void addToken(int type, int line, int start, int length) {
-        tokens.add(new RustToken(type, currentFile, line, start, length));
+    /* package-private */ void addToken(RustTokenType type, int line, int start, int length) {
+        tokens.add(new Token(type, currentFile, line, start, length));
 
     }
 }

languages/rust/src/main/java/de/jplag/rust/RustToken.java

@@ -1,129 +0,0 @@
-package de.jplag.rust;
-
-import static de.jplag.rust.RustTokenConstants.*;
-
-import de.jplag.Token;
-
-public class RustToken extends Token {
-    public RustToken(int type, String currentFile, int line, int start, int length) {
-        super(type, currentFile, line, start, length);
-    }
-
-    @Override
-    protected String type2string() {
-        return switch (type) {
-            case FILE_END -> "<EOF>";
-            case SEPARATOR_TOKEN -> "--------";
-            case INNER_ATTRIBUTE -> "INNER_ATTR";
-            case OUTER_ATTRIBUTE -> "OUTER_ATTR";
-
-            case USE_DECLARATION -> "USE";
-            case USE_ITEM -> "USE_ITEM";
-
-            case MODULE -> "MODULE";
-            case MODULE_START -> "MODULE{";
-            case MODULE_END -> "}MODULE";
-
-            case FUNCTION -> "FUNCTION";
-            case TYPE_PARAMETER -> "<T>";
-            case FUNCTION_PARAMETER -> "PARAM";
-            case FUNCTION_BODY_START -> "FUNC{";
-            case FUNCTION_BODY_END -> "}FUNC";
-
-            case STRUCT -> "STRUCT";
-            case STRUCT_BODY_START -> "STRUCT{";
-            case STRUCT_BODY_END -> "}STRUCT";
-            case STRUCT_INITIALISATION -> "STRUCT()";
-
-            case STRUCT_FIELD -> "FIELD";
-
-            case UNION -> "UNION";
-            case UNION_BODY_START -> "UNION{";
-            case UNION_BODY_END -> "}UNION";
-
-            case TRAIT -> "TRAIT";
-            case TRAIT_BODY_START -> "TRAIT{";
-            case TRAIT_BODY_END -> "}TRAIT";
-
-            case IMPLEMENTATION -> "IMPL";
-            case IMPLEMENTATION_BODY_START -> "IMPL{";
-            case IMPLEMENTATION_BODY_END -> "}IMPL";
-
-            case ENUM -> "ENUM";
-            case ENUM_BODY_START -> "ENUM{";
-            case ENUM_BODY_END -> "}ENUM";
-            case ENUM_ITEM -> "ENUM_ITEM";
-
-            case MACRO_RULES_DEFINITION -> "MACRO_RULES";
-            case MACRO_RULES_DEFINITION_BODY_START -> "MACRO_RULES{";
-            case MACRO_RULES_DEFINITION_BODY_END -> "}MACRO_RULES";
-
-            case MACRO_RULE -> "MACRO_RULE";
-            case MACRO_RULE_BODY_START -> "MACRO_RULE{";
-            case MACRO_RULE_BODY_END -> "}MACRO_RULE";
-
-            case MACRO_INVOCATION -> "MACRO()";
-            case MACRO_INVOCATION_BODY_START -> "MACRO(){";
-            case MACRO_INVOCATION_BODY_END -> "}MACRO()";
-
-            case EXTERN_BLOCK -> "EXTERN";
-            case EXTERN_BLOCK_START -> "EXTERN{";
-            case EXTERN_BLOCK_END -> "}EXTERN";
-            case TYPE_ALIAS -> "TYPE_ALIAS";
-            case STATIC_ITEM -> "STATIC";
-
-            case EXTERN_CRATE -> "EXTERN";
-
-            case IF_STATEMENT -> "IF";
-            case IF_BODY_START -> "IF{";
-            case IF_BODY_END -> "}IF";
-            case ELSE_STATEMENT -> "ELSE";
-            case ELSE_BODY_START -> "ELSE{";
-            case ELSE_BODY_END -> "ELSE}";
-
-            case LABEL -> "LABEL";
-            case LOOP_STATEMENT -> "LOOP";
-            case LOOP_BODY_START -> "LOOP{";
-            case LOOP_BODY_END -> "}LOOP";
-            case FOR_STATEMENT -> "FOR";
-            case FOR_BODY_START -> "FOR{";
-            case FOR_BODY_END -> "}FOR";
-
-            case BREAK -> "BREAK";
-
-            case MATCH_EXPRESSION -> "MATCH";
-            case MATCH_BODY_START -> "MATCH{";
-            case MATCH_BODY_END -> "}MATCH";
-            case MATCH_CASE -> "CASE";
-            case MATCH_GUARD -> "GUARD";
-
-            case INNER_BLOCK_START -> "INNER{";
-            case INNER_BLOCK_END -> "}INNER";
-
-            case ARRAY_BODY_START -> "ARRAY{";
-            case ARRAY_BODY_END -> "}ARRAY";
-            case ARRAY_ELEMENT -> "ARRAY_ELEM";
-
-            case TUPLE -> "TUPLE";
-            case TUPLE_START -> "TUPLE(";
-            case TUPLE_END -> ")TUPLE";
-            case TUPLE_ELEMENT -> "T_ELEM";
-
-            case CLOSURE -> "CLOSURE";
-            case CLOSURE_BODY_START -> "CLOSURE{";
-            case CLOSURE_BODY_END -> "}CLOSURE";
-
-            case APPLY -> "APPLY";
-            case ARGUMENT -> "ARG";
-            case ASSIGNMENT -> "ASSIGN";
-
-            case VARIABLE_DECLARATION -> "VAR_DECL";
-
-            case TYPE_ARGUMENT -> "T_ARG";
-
-            case RETURN -> "RETURN";
-
-            default -> "<UNKNOWN%d>".formatted(type);
-        };
-    }
-}

languages/rust/src/main/java/de/jplag/rust/RustTokenConstants.java

@@ -1,118 +0,0 @@
-package de.jplag.rust;
-
-import de.jplag.TokenConstants;
-
-public interface RustTokenConstants extends TokenConstants {
-
-    int INNER_ATTRIBUTE = 2;
-    int OUTER_ATTRIBUTE = 3;
-
-    int USE_DECLARATION = 4;
-    int USE_ITEM = 5;
-
-    int MODULE = 6;
-    int MODULE_START = 7;
-    int MODULE_END = 8;
-
-    int FUNCTION = 9;
-    int TYPE_PARAMETER = 10;
-    int FUNCTION_PARAMETER = 11;
-    int FUNCTION_BODY_START = 12;
-    int FUNCTION_BODY_END = 13;
-
-    int STRUCT = 14;
-    int STRUCT_BODY_START = 15;
-    int STRUCT_BODY_END = 16;
-    int STRUCT_INITIALISATION = 17;
-
-    int STRUCT_FIELD = 18;
-
-    int UNION = 19;
-    int UNION_BODY_START = 20;
-    int UNION_BODY_END = 21;
-
-    int TRAIT = 22;
-    int TRAIT_BODY_START = 23;
-    int TRAIT_BODY_END = 24;
-
-    int IMPLEMENTATION = 25;
-    int IMPLEMENTATION_BODY_START = 26;
-    int IMPLEMENTATION_BODY_END = 27;
-
-    int ENUM = 28;
-    int ENUM_BODY_START = 29;
-    int ENUM_BODY_END = 30;
-    int ENUM_ITEM = 31;
-
-    int MACRO_RULES_DEFINITION = 32;
-    int MACRO_RULES_DEFINITION_BODY_START = 33;
-    int MACRO_RULES_DEFINITION_BODY_END = 34;
-
-    int MACRO_RULE = 35;
-    int MACRO_RULE_BODY_START = 36;
-    int MACRO_RULE_BODY_END = 37;
-
-    int MACRO_INVOCATION = 38;
-    int MACRO_INVOCATION_BODY_START = 39;
-    int MACRO_INVOCATION_BODY_END = 40;
-
-    int EXTERN_BLOCK = 41;
-    int EXTERN_BLOCK_START = 42;
-    int EXTERN_BLOCK_END = 43;
-    int TYPE_ALIAS = 44;
-    int STATIC_ITEM = 45;
-
-    int EXTERN_CRATE = 46;
-
-    int IF_STATEMENT = 47;
-    int IF_BODY_START = 48;
-    int IF_BODY_END = 49;
-    int ELSE_STATEMENT = 50;
-    int ELSE_BODY_START = 51;
-    int ELSE_BODY_END = 52;
-
-    int LABEL = 53;
-    int LOOP_STATEMENT = 54;
-    int LOOP_BODY_START = 55;
-    int LOOP_BODY_END = 56;
-    int FOR_STATEMENT = 57;
-    int FOR_BODY_START = 58;
-    int FOR_BODY_END = 59;
-
-    int BREAK = 60;
-
-    int MATCH_EXPRESSION = 61;
-    int MATCH_BODY_START = 62;
-    int MATCH_BODY_END = 63;
-    int MATCH_CASE = 64;
-    int MATCH_GUARD = 65;
-
-    int INNER_BLOCK_START = 66;
-    int INNER_BLOCK_END = 67;
-
-    int ARRAY_BODY_START = 68;
-    int ARRAY_BODY_END = 69;
-    int ARRAY_ELEMENT = 70;
-
-    int TUPLE = 71;
-    int TUPLE_START = 72;
-    int TUPLE_END = 73;
-    int TUPLE_ELEMENT = 74;
-
-    int CLOSURE = 75;
-    int CLOSURE_BODY_START = 76;
-    int CLOSURE_BODY_END = 77;
-
-    int APPLY = 78;
-    int ARGUMENT = 79;
-    int ASSIGNMENT = 80;
-
-    int VARIABLE_DECLARATION = 81;
-
-    int TYPE_ARGUMENT = 82;
-
-    int RETURN = 83;
-
-    int NUMBER_DIFF_TOKENS = 84;
-
-}

languages/rust/src/main/java/de/jplag/rust/RustTokenType.java

@@ -0,0 +1,125 @@
+package de.jplag.rust;
+
+import de.jplag.TokenType;
+
+public enum RustTokenType implements TokenType {
+    INNER_ATTRIBUTE("INNER_ATTR"),
+    OUTER_ATTRIBUTE("OUTER_ATTR"),
+
+    USE_DECLARATION("USE"),
+    USE_ITEM("USE_ITEM"),
+
+    MODULE("MODULE"),
+    MODULE_START("MODULE{"),
+    MODULE_END("}MODULE"),
+
+    FUNCTION("FUNCTION"),
+    TYPE_PARAMETER("<T>"),
+    FUNCTION_PARAMETER("PARAM"),
+    FUNCTION_BODY_START("FUNC{"),
+    FUNCTION_BODY_END("}FUNC"),
+
+    STRUCT("STRUCT"),
+    STRUCT_BODY_START("STRUCT{"),
+    STRUCT_BODY_END("}STRUCT"),
+    STRUCT_INITIALISATION("STRUCT()"),
+
+    STRUCT_FIELD("FIELD"),
+
+    UNION("UNION"),
+    UNION_BODY_START("UNION{"),
+    UNION_BODY_END("}UNION"),
+
+    TRAIT("TRAIT"),
+    TRAIT_BODY_START("TRAIT{"),
+    TRAIT_BODY_END("}TRAIT"),
+
+    IMPLEMENTATION("IMPL"),
+    IMPLEMENTATION_BODY_START("IMPL{"),
+    IMPLEMENTATION_BODY_END("}IMPL"),
+
+    ENUM("ENUM"),
+    ENUM_BODY_START("ENUM{"),
+    ENUM_BODY_END("}ENUM"),
+    ENUM_ITEM("ENUM_ITEM"),
+
+    MACRO_RULES_DEFINITION("MACRO_RULES"),
+    MACRO_RULES_DEFINITION_BODY_START("MACRO_RULES{"),
+    MACRO_RULES_DEFINITION_BODY_END("}MACRO_RULES"),
+
+    MACRO_RULE("MACRO_RULE"),
+    MACRO_RULE_BODY_START("MACRO_RULE{"),
+    MACRO_RULE_BODY_END("}MACRO_RULE"),
+
+    MACRO_INVOCATION("MACRO()"),
+    MACRO_INVOCATION_BODY_START("MACRO(){"),
+    MACRO_INVOCATION_BODY_END("}MACRO()"),
+
+    EXTERN_BLOCK("EXTERN"),
+    EXTERN_BLOCK_START("EXTERN{"),
+    EXTERN_BLOCK_END("}EXTERN"),
+    TYPE_ALIAS("TYPE_ALIAS"),
+    STATIC_ITEM("STATIC"),
+
+    EXTERN_CRATE("EXTERN"),
+
+    IF_STATEMENT("IF"),
+    IF_BODY_START("IF{"),
+    IF_BODY_END("}IF"),
+    ELSE_STATEMENT("ELSE"),
+    ELSE_BODY_START("ELSE{"),
+    ELSE_BODY_END("ELSE}"),
+
+    LABEL("LABEL"),
+    LOOP_STATEMENT("LOOP"),
+    LOOP_BODY_START("LOOP{"),
+    LOOP_BODY_END("}LOOP"),
+    FOR_STATEMENT("FOR"),
+    FOR_BODY_START("FOR{"),
+    FOR_BODY_END("}FOR"),
+
+    BREAK("BREAK"),
+
+    MATCH_EXPRESSION("MATCH"),
+    MATCH_BODY_START("MATCH{"),
+    MATCH_BODY_END("}MATCH"),
+    MATCH_CASE("CASE"),
+    MATCH_GUARD("GUARD"),
+
+    INNER_BLOCK_START("INNER{"),
+    INNER_BLOCK_END("}INNER"),
+
+    ARRAY_BODY_START("ARRAY{"),
+    ARRAY_BODY_END("}ARRAY"),
+    ARRAY_ELEMENT("ARRAY_ELEM"),
+
+    TUPLE("TUPLE"),
+    TUPLE_START("TUPLE("),
+    TUPLE_END(")TUPLE"),
+    TUPLE_ELEMENT("T_ELEM"),
+
+    CLOSURE("CLOSURE"),
+    CLOSURE_BODY_START("CLOSURE{"),
+    CLOSURE_BODY_END("}CLOSURE"),
+
+    APPLY("APPLY"),
+    ARGUMENT("ARG"),
+    ASSIGNMENT("ASSIGN"),
+
+    VARIABLE_DECLARATION("VAR_DECL"),
+
+    TYPE_ARGUMENT("T_ARG"),
+
+    RETURN("RETURN");
+
+    private final String description;
+
+    public String getDescription() {
+        return description;
+    }
+
+    RustTokenType(String description) {
+        this.description = description;
+    }
+
+}

languages/rust/src/test/java/de/jplag/rust/RustFrontendTest.java

@@ -7,7 +7,9 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import org.junit.jupiter.api.BeforeEach;
@@ -15,8 +17,8 @@ import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import de.jplag.SharedTokenType;
 import de.jplag.Token;
-import de.jplag.TokenConstants;
 import de.jplag.TokenPrinter;
 
 class RustFrontendTest {
@@ -32,8 +34,6 @@ class RustFrontendTest {
      * Test source file that is supposed to produce a complete set of tokens, i.e. all types of tokens.
      */
     private static final String COMPLETE_TEST_FILE = "complete.rs";
-    public static final int NOT_SET = -1;
-    private static final String EMPTY_STRING = "";
     private static final String RUST_SHEBANG = "#!.*$";
     private static final double EPSILON = 1E-6;
     public static final double BASELINE_COVERAGE = 0.75;
@@ -75,7 +75,7 @@ class RustFrontendTest {
             // All lines that contain code
             var codeLines = new ArrayList<>(getCodeLines(lines));
             // All lines that contain token
-            var tokenLines = tokens.stream().map(Token::getLine).filter(line -> line != Token.NO_VALUE).distinct().toList();
+            var tokenLines = tokens.stream().mapToInt(Token::getLine).filter(line -> line != Token.NO_VALUE).distinct().boxed().toList();
 
             // Keep only lines that have no tokens
             codeLines.removeAll(tokenLines);
@@ -125,21 +125,13 @@ class RustFrontendTest {
      * @param fileName The file name of the complete code example
      */
     private void testTokenCoverage(List<Token> tokens, String fileName) {
-        var foundTokens = tokens.stream().map(Token::getType).distinct().toList();
-        var allTokens = IntStream.range(0, RustTokenConstants.NUMBER_DIFF_TOKENS).boxed().toList();
-        allTokens = new ArrayList<>(allTokens);
-
-        // Only non-found tokens are left
-        allTokens.removeAll(foundTokens);
-        // Exclude SEPARATOR_TOKEN, as it does not occur
-        allTokens.remove((Integer) (TokenConstants.SEPARATOR_TOKEN));
-
-        if (!allTokens.isEmpty()) {
-            var notFoundTypes = allTokens.stream().map(type -> new RustToken(type, EMPTY_STRING, NOT_SET, NOT_SET, NOT_SET).type2string()).toList();
-            logger.error("Some %d token types were not found in the complete code example '%s':\n%s".formatted(notFoundTypes.size(), fileName,
-                    notFoundTypes));
-            assertTrue(false);
-        }
+        var annotatedTokens = tokens.stream().map(Token::getType).collect(Collectors.toSet());
+        assertTrue(annotatedTokens.contains(SharedTokenType.FILE_END));
+        var annotatedRustTokens = annotatedTokens.stream().filter(RustTokenType.class::isInstance).collect(Collectors.toSet());
+        var allRustTokens = RustTokenType.values();
+        var missingRustTokens = Arrays.stream(allRustTokens).filter(token -> !annotatedRustTokens.contains(token)).toList();
+        assertTrue(missingRustTokens.isEmpty(), "The following rust tokens are missing in the code example '%s':\n".formatted(fileName)
+                + String.join("\n", missingRustTokens.stream().map(RustTokenType::getDescription).toList()));
     }
 
 }

languages/scala/README.md

@@ -20,7 +20,7 @@ so the upgrade needs to wait. It would seem that once this frontend is equipped
 The choice of tokens is intended to be similar to the Java or C# frontends. Specifically, among others, it includes a
 range of nesting structures (class and method declarations, control flow expressions) as well as variable declaration,
 object creation, assignment, and control flow altering keywords. <br>
-Blocks are distinguished by their context, i.e. there are separate `TokenConstants` for `if` blocks, `for` blocks, class
+Blocks are distinguished by their context, i.e. there are separate `TokenTypes` for `if` blocks, `for` blocks, class
 bodies, method bodies, array constructors, and the like.
 
 More syntactic elements of Scala may turn out to be helpful to include in the future, especially those that are newly

languages/scala/src/main/scala/de/jplag/scala/Language.scala

@@ -3,10 +3,10 @@ package de.jplag.scala
 import de.jplag.Token
 
 import java.io.File
-import scala.collection.JavaConverters._
-
 import org.kohsuke.MetaInfServices
 
+import scala.jdk.CollectionConverters.SeqHasAsJava
+
 class Language extends de.jplag.Language {
   private val parser = new Parser
   private final val fileExtensions = Array(".scala", ".sc")

languages/scala/src/main/scala/de/jplag/scala/Parser.scala

@@ -1,6 +1,6 @@
 package de.jplag.scala
 
-import de.jplag.scala.ScalaTokenConstants._
+import de.jplag.scala.ScalaTokenType._
 import de.jplag.{AbstractParser, Token}
 
 import java.io.File
@@ -33,8 +33,8 @@ class Parser extends AbstractParser {
          * @param after    opt. token type to insert at the end of the tree
          * @param traverse custom method of traversing the tree
          */
-        private case class TR(before: Option[ScalaTokenConstants.Value] = None,
-                              after: Option[ScalaTokenConstants.Value] = None,
+        private case class TR(before: Option[ScalaTokenType] = None,
+                              after: Option[ScalaTokenType] = None,
                               traverse: Tree => Unit = _.children.foreach(traverser.apply)
                              )
 
@@ -44,7 +44,7 @@ class Parser extends AbstractParser {
          * @param tree  opt. tree
          * @param token token type to insert at the beginning of the tree
          */
-        private def maybeAddAndApply(tree: Option[Tree], token: ScalaTokenConstants.Value): Unit = tree match {
+        private def maybeAddAndApply(tree: Option[Tree], token: ScalaTokenType): Unit = tree match {
             case Some(expression) =>
                 add(token, expression, fromEnd = false)
                 apply(expression)
@@ -53,10 +53,10 @@ class Parser extends AbstractParser {
 
         private def processCases(cases: List[Case]): Unit = cases.foreach {
             case caseTree@Case(pattern, condition, body) =>
-                add(ScalaTokenConstants.CaseStatement, caseTree, fromEnd = false)
+                add(CASE_STATEMENT, caseTree, fromEnd = false)
                 applyRecursively(Seq(pattern, condition))
 
-                encloseAndApply(body, TR(Some(CaseBegin), Some(CaseEnd)))
+                encloseAndApply(body, TR(Some(CASE_BEGIN), Some(CASE_END)))
             case _ =>
         }
 
@@ -71,7 +71,7 @@ class Parser extends AbstractParser {
         private def handleDefinitionPattern(pattern: Pat, optionalValue: Option[Term]): Unit = {
             pattern match {
                 // variable tuple
-                case tuple@Pat.Tuple(patternArgs) =>
+                case Pat.Tuple(patternArgs) =>
                     optionalValue match {
                         // not initialized
                         case None => for (elem <- patternArgs) handleDefinitionPattern(elem, None)
@@ -84,34 +84,34 @@ class Parser extends AbstractParser {
                         // initialized with tuple reference
                         case Some(value) =>
                             for (elem <- patternArgs) handleDefinitionPattern(elem, None)
-                            add(Assign, value, fromEnd = false)
+                            add(ASSIGN, value, fromEnd = false)
                     }
 
                 // single variable
                 case other@_ =>
-                    add(VariableDefinition, other, fromEnd = false)
+                    add(VARIABLE_DEFINITION, other, fromEnd = false)
                     apply(other)
-                    maybeAddAndApply(optionalValue, Assign)
+                    maybeAddAndApply(optionalValue, ASSIGN)
             }
         }
 
         private def doMatch(tree: Tree): TR = {
             tree match {
-                case Term.Do(body, condition) => TR(Some(DoWhile), Some(DoWhileEnd), traverse = _ => {
-                    encloseAndApply(body, TR(Some(DoBodyBegin), Some(DoBodyEnd)))
+                case Term.Do(body, condition) => TR(Some(DO_WHILE), Some(DO_WHILE_END), traverse = _ => {
+                    encloseAndApply(body, TR(Some(DO_BODY_BEGIN), Some(DO_BODY_END)))
                     apply(condition)
                 })
-                case Term.Assign(_) => TR(Some(Assign))
-                case Term.While(condition, body) => TR(Some(While), traverse = _ => {
+                case Term.Assign(_) => TR(Some(ASSIGN))
+                case Term.While(condition, body) => TR(Some(WHILE), traverse = _ => {
                     apply(condition)
-                    encloseAndApply(body, TR(Some(WhileBodyBegin), Some(WhileBodyEnd)))
+                    encloseAndApply(body, TR(Some(WHILE_BODY_BEGIN), Some(WHILE_BODY_END)))
                 })
                 case Term.For(enumerators, body) =>
-                    TR(Some(For), traverse = _ => {
+                    TR(Some(FOR), traverse = _ => {
                         apply(enumerators)
-                        encloseAndApply(body, TR(Some(ForBodyBegin), Some(ForBodyEnd)))
+                        encloseAndApply(body, TR(Some(FOR_BODY_BEGIN), Some(FOR_BODY_END)))
                     })
-                case Term.Try(expression, catchExpression, finallyExpression) => TR(Some(TryBegin),
+                case Term.Try(expression, catchExpression, finallyExpression) => TR(Some(TRY_BEGIN),
                     traverse = _ => {
                         apply(expression)
                         if (catchExpression.nonEmpty) {
@@ -119,28 +119,28 @@ class Parser extends AbstractParser {
                             val end = catchExpression.last.pos
                             val length = end.endLine - start.startLine
 
-                            add(CatchBegin, start.startLine, start.startColumn, length)
+                            add(CATCH_BEGIN, start.startLine, start.startColumn, length)
                             processCases(catchExpression)
-                            add(CatchEnd, end.endLine, end.endColumn, length)
+                            add(CATCH_END, end.endLine, end.endColumn, length)
                         }
 
-                        maybeAddAndApply(finallyExpression, ScalaTokenConstants.Finally)
+                        maybeAddAndApply(finallyExpression, FINALLY)
                     })
-                case Term.TryWithHandler(expression, catchExpression, finallyExpression) => TR(Some(TryBegin),
+                case Term.TryWithHandler(expression, catchExpression, finallyExpression) => TR(Some(TRY_BEGIN),
                     traverse = _ => {
                         apply(expression)
-                        encloseAndApply(catchExpression, TR(Some(CatchBegin), Some(CatchEnd)))
+                        encloseAndApply(catchExpression, TR(Some(CATCH_BEGIN), Some(CATCH_END)))
 
-                        maybeAddAndApply(finallyExpression, ScalaTokenConstants.Finally)
+                        maybeAddAndApply(finallyExpression, FINALLY)
                     })
                 case Term.Apply(function, arguments) if !isStandardOperator(getMethodIdentifier(function)) && arguments.nonEmpty =>
                     // function calls with no arguments are not covered here; see README
                     TR(traverse = _ => {
 
-                        add(Apply, function, fromEnd = false)
+                        add(APPLY, function, fromEnd = false)
                         apply(function)
                         for (argument <- arguments) {
-                            add(Argument, argument, fromEnd = false)
+                            add(ARGUMENT, argument, fromEnd = false)
                             argument match {
                                 case Term.Assign(assignee, value) =>
                                     // default values
@@ -150,59 +150,59 @@ class Parser extends AbstractParser {
                             }
                         }
                     })
-                case Term.NewAnonymous(_) => TR(Some(NewCreationBegin), Some(NewCreationEnd))
-                case Term.Return(_) => TR(Some(ScalaTokenConstants.Return))
-                case Term.Match(expression, cases) => TR(Some(MatchBegin), Some(MatchEnd), traverse = _ => {
+                case Term.NewAnonymous(_) => TR(Some(NEW_CREATION_BEGIN), Some(NEW_CREATION_END))
+                case Term.Return(_) => TR(Some(RETURN))
+                case Term.Match(expression, cases) => TR(Some(MATCH_BEGIN), Some(MATCH_END), traverse = _ => {
                     apply(expression)
                     processCases(cases)
                 })
-                case Term.Throw(_) => TR(Some(Throw))
-                case Term.Function(_) => TR(Some(FunctionBegin), Some(FunctionEnd))
-                case Term.PartialFunction(cases) => TR(Some(PartialFunctionBegin), Some(PartialFunctionEnd), traverse = _ => {
+                case Term.Throw(_) => TR(Some(THROW))
+                case Term.Function(_) => TR(Some(FUNCTION_BEGIN), Some(FUNCTION_END))
+                case Term.PartialFunction(cases) => TR(Some(PARTIAL_FUNCTION_BEGIN), Some(PARTIAL_FUNCTION_END), traverse = _ => {
                     processCases(cases)
                 })
                 case Term.ForYield(enumerators, body) => TR(traverse = _ => {
                     apply(enumerators)
-                    add(ForBodyBegin, body, fromEnd = false)
-                    encloseAndApply(body, TR(Some(Yield), Some(ForBodyEnd)))
+                    add(FOR_BODY_BEGIN, body, fromEnd = false)
+                    encloseAndApply(body, TR(Some(YIELD), Some(FOR_BODY_END)))
                 })
                 case Term.If(condition, thenExpression, elseExpression) => TR(traverse = _ => {
-                    add(If, tree, fromEnd = false)
+                    add(IF, tree, fromEnd = false)
                     apply(condition)
 
-                    encloseAndApply(thenExpression, TR(Some(IfBegin), Some(IfEnd)))
+                    encloseAndApply(thenExpression, TR(Some(IF_BEGIN), Some(IF_END)))
 
                     elseExpression match {
                         case Lit.Unit() => apply(elseExpression)
                         case _ =>
                             val elseStart = tree.pos.text.indexOf("else", thenExpression.pos.end - tree.pos.start)
                             val elsePosition = Position.Range(tree.pos.input, tree.pos.start + elseStart, tree.pos.start + elseStart + 4)
-                            add(Else, elsePosition.startLine + 1, elsePosition.startColumn + 1, elsePosition.text.length)
-                            encloseAndApply(elseExpression, TR(Some(ElseBegin), Some(ElseEnd)))
+                            add(ELSE, elsePosition.startLine + 1, elsePosition.startColumn + 1, elsePosition.text.length)
+                            encloseAndApply(elseExpression, TR(Some(ELSE_BEGIN), Some(ELSE_END)))
                     }
                 })
 
-                case scala.meta.Pkg(_) => TR(Some(Package))
-                case scala.meta.Import(_) => TR(Some(ScalaTokenConstants.Import))
+                case scala.meta.Pkg(_) => TR(Some(PACKAGE))
+                case scala.meta.Import(_) => TR(Some(IMPORT))
 
                 case Defn.Def(modifiers, name, typeParameters, parameterLists, _, body) =>
                     TR(traverse = _ => {
                         applyRecursively(modifiers)
-                        add(MethodDef, name, fromEnd = false)
-                        assignRecursively(typeParameters, TypeParameter)
-                        assignRecursively(parameterLists, Parameter)
+                        add(METHOD_DEF, name, fromEnd = false)
+                        assignRecursively(typeParameters, TYPE_PARAMETER)
+                        assignRecursively(parameterLists, PARAMETER)
 
-                        encloseAndApply(body, TR(Some(MethodBegin), Some(MethodEnd)))
+                        encloseAndApply(body, TR(Some(METHOD_BEGIN), Some(METHOD_END)))
                     })
-                case Defn.Macro(modifiers, macroName, typeParameters, parameterLists, declaredType, body) => TR(Some(Macro), traverse = _ => {
+                case Defn.Macro(modifiers, _, typeParameters, parameterLists, _, body) => TR(Some(MACRO), traverse = _ => {
                     applyRecursively(Seq(modifiers, typeParameters, parameterLists))
-                    encloseAndApply(body, TR(Some(MacroBegin), Some(MacroEnd)))
+                    encloseAndApply(body, TR(Some(MACRO_BEGIN), Some(MACRO_END)))
                 })
                 case Defn.Class(_) =>
-                    TR(Some(ClassBegin), Some(ClassEnd))
-                case Defn.Object(_) => TR(Some(ObjectBegin), Some(ObjectEnd))
-                case Defn.Trait(_) => TR(Some(TraitBegin), Some(TraitEnd))
-                case Defn.Type(_) => TR(Some(ScalaTokenConstants.Type))
+                    TR(Some(CLASS_BEGIN), Some(CLASS_END))
+                case Defn.Object(_) => TR(Some(OBJECT_BEGIN), Some(OBJECT_END))
+                case Defn.Trait(_) => TR(Some(TRAIT_BEGIN), Some(TRAIT_END))
+                case Defn.Type(_) => TR(Some(TYPE))
                 case Defn.Var(modifiers, patterns, declaredType, optionalValue) => TR(traverse = _ => {
                     apply(modifiers)
                     for (pattern <- patterns) {
@@ -219,47 +219,47 @@ class Parser extends AbstractParser {
 
                 })
 
-                case Decl.Var(_) => TR(Some(VariableDefinition))
-                case Decl.Val(_) => TR(Some(VariableDefinition))
-                case Decl.Def(_) => TR(Some(MethodBegin), Some(MethodEnd))
-                case Decl.Type(_) => TR(Some(ScalaTokenConstants.Type))
+                case Decl.Var(_) => TR(Some(VARIABLE_DEFINITION))
+                case Decl.Val(_) => TR(Some(VARIABLE_DEFINITION))
+                case Decl.Def(_) => TR(Some(METHOD_BEGIN), Some(METHOD_END))
+                case Decl.Type(_) => TR(Some(TYPE))
 
                 case Ctor.Secondary(_) =>
-                    TR(Some(ConstructorBegin), Some(ConstructorEnd))
+                    TR(Some(CONSTRUCTOR_BEGIN), Some(CONSTRUCTOR_END))
 
-                case Init(typeName, name, argumentLists) if argumentLists.nonEmpty => TR(traverse = _ => {
-                    assignRecursively(argumentLists, Argument, doApply = true)
+                case Init(_, _, argumentLists) if argumentLists.nonEmpty => TR(traverse = _ => {
+                    assignRecursively(argumentLists, ARGUMENT, doApply = true)
                 })
-                case Enumerator.Guard(_) => TR(Some(Guard))
+                case Enumerator.Guard(_) => TR(Some(GUARD))
 
-                case Term.Param(_) => TR(traverse = _ => add(Parameter, tree, fromEnd = false))
-                case Term.ApplyInfix(_, operator, _, _) if operator.value.contains("=") && !Array("==", "!=").contains(operator.value) => TR(Some(Assign))
+                case Term.Param(_) => TR(traverse = _ => add(PARAMETER, tree, fromEnd = false))
+                case Term.ApplyInfix(_, operator, _, _) if operator.value.contains("=") && !Array("==", "!=").contains(operator.value) => TR(Some(ASSIGN))
                 case Term.ApplyInfix(function, operator, typeArgs, arguments) if !isStandardOperator(operator.value) => TR(traverse = _ => {
-                    add(Apply, tree, fromEnd = false)
+                    add(APPLY, tree, fromEnd = false)
                     apply(function)
-                    assignRecursively(typeArgs, TypeArgument, doApply = true)
-                    assignRecursively(arguments, Argument, doApply = true)
+                    assignRecursively(typeArgs, TYPE_ARGUMENT, doApply = true)
+                    assignRecursively(arguments, ARGUMENT, doApply = true)
                 })
                 case Term.Select(refObj, member) =>
                     TR(traverse = _ => {
                         apply(refObj)
-                        if (!isStandardOperator(member.value)) add(ScalaTokenConstants.Member, member, fromEnd = false)
+                        if (!isStandardOperator(member.value)) add(MEMBER, member, fromEnd = false)
                         apply(member)
                     })
                 case Term.ApplyType(_, typeArgs) => TR(traverse = _ => {
-                    add(Apply, tree, fromEnd = false)
-                    assignRecursively(typeArgs, TypeArgument)
+                    add(APPLY, tree, fromEnd = false)
+                    assignRecursively(typeArgs, TYPE_ARGUMENT)
                 })
-                case Term.New(_) => TR(Some(NewObject))
-                case Self(_) => TR(Some(SelfType))
+                case Term.New(_) => TR(Some(NEW_OBJECT))
+                case Self(_) => TR(Some(SELF_TYPE))
                 case block@Term.Block(_) => block.parent match {
                     // inner block
-                    case Some(Term.Apply(_)) => TR(Some(BlockStart), Some(BlockEnd))
+                    case Some(Term.Apply(_)) => TR(Some(BLOCK_START), Some(BLOCK_END))
                     // block in an expression context, e.g. for, if, while
                     case _ => TR()
                 }
-                case Enumerator.Generator(_) => TR(Some(EnumGenerator))
-                case meta.Type.Param(_) => TR(Some(TypeParameter))
+                case Enumerator.Generator(_) => TR(Some(ENUM_GENERATOR))
+                case meta.Type.Param(_) => TR(Some(TYPE_PARAMETER))
 
                 case _ => TR()
             }
@@ -301,7 +301,7 @@ class Parser extends AbstractParser {
          * @param doApply   if true, the substructure of each element is traversed
          * @tparam T list may contain more lists or leaves
          */
-        private def assignRecursively[T](els: List[T], tokenType: ScalaTokenConstants.Value, doApply: Boolean = false): Unit = els.foreach {
+        private def assignRecursively[T](els: List[T], tokenType: ScalaTokenType, doApply: Boolean = false): Unit = els.foreach {
             case treeList: List[_] => assignRecursively(treeList, tokenType)
             case el: Tree =>
                 add(tokenType, el, fromEnd = false)
@@ -356,7 +356,7 @@ class Parser extends AbstractParser {
             val ast = input.parse[Source].get
             traverser(ast)
 
-            add(FileEnd, text.count(_ == '\n') - 1, 0, 0)
+            tokens += Token.fileEnd(currentFile)
         } catch {
             case exception: Throwable =>
                 exception.printStackTrace()
@@ -374,8 +374,8 @@ class Parser extends AbstractParser {
      * @param column    column of the occurrence in the file
      * @param length    length of the occurrence in the file
      */
-    private def add(tokenType: ScalaTokenConstants.Value, line: Int, column: Int, length: Int): Unit = {
-        tokens += new ScalaToken(tokenType.id, currentFile, line, column, length)
+    private def add(tokenType: ScalaTokenType, line: Int, column: Int, length: Int): Unit = {
+        tokens += new Token(tokenType, currentFile, line, column, length)
     }
 
 
@@ -386,10 +386,15 @@ class Parser extends AbstractParser {
      * @param node      the tree that marks the occurrence
      * @param fromEnd   if true, the token is added at the end of the tree (length 0).
      */
-    private def add(tokenType: ScalaTokenConstants.Value, node: Tree, fromEnd: Boolean): Unit = {
+    private def add(tokenType: ScalaTokenType, node: Tree, fromEnd: Boolean): Unit = {
         if (node.pos.text.nonEmpty) {
             // SELF type tokens with no text content mess up the sequence
-            tokens += new ScalaToken(tokenType, currentFile, node.pos, fromEnd)
+            if (fromEnd) {
+                tokens += new Token(tokenType, currentFile, node.pos.endLine + 1, node.pos.endColumn + 1, 0)
+            }
+            else {
+                tokens += new Token(tokenType, currentFile, node.pos.startLine + 1, node.pos.endColumn + 1, node.pos.text.length)
+            }
         }
     }
 

languages/scala/src/main/scala/de/jplag/scala/ScalaToken.scala

@@ -1,36 +0,0 @@
-package de.jplag.scala
-
-import de.jplag.Token
-
-import scala.meta.inputs.Position
-
-class ScalaToken(tokenType: Int,
-                 file: java.lang.String,
-                 var line: Int,
-                 var column: Int,
-                 var length: Int)
-  extends Token(tokenType, file, line, column, length) {
-
-    def this(tokenType: ScalaTokenConstants.Value, file: String, position: Position, fromEnd: Boolean) {
-        this(
-            tokenType.id,
-            file,
-            (if (fromEnd) position.endLine else position.startLine) + 1,
-            (if (fromEnd) position.endColumn else position.startColumn) + 1,
-            if (fromEnd) 0 else position.text.length,
-        )
-    }
-
-    override def toString: String = {
-        try {
-            ScalaTokenConstants(`type`).toString
-        } catch {
-            case _: Throwable => "Unknown!"
-        }
-    }
-
-    /**
-     * @return a string representation depending on the type of the token.
-     */
-    override protected def type2string(): String = ScalaTokenConstants.apply(tokenType).toString
-}

languages/scala/src/main/scala/de/jplag/scala/ScalaTokenConstants.scala

@@ -1,83 +0,0 @@
-package de.jplag.scala
-
-import de.jplag.TokenConstants
-
-/**
- * This Java-compatible enumeration stores the possible types of tokens.
- */
-object ScalaTokenConstants extends Enumeration with TokenConstants {
-
-  val FileEnd: Value = Value(TokenConstants.FILE_END, "<EOF>")
-  val SeparatorToken: Value = Value(TokenConstants.SEPARATOR_TOKEN, "--------")
-  val Package: Value = Value("PACKAGE")
-  val Import: Value = Value("IMPORT")
-  val ClassBegin: Value = Value("CLASS{")
-  val ClassEnd: Value = Value("}CLASS")
-  val MethodDef: Value = Value("METHOD")
-  val MethodBegin: Value = Value("METHOD{")
-  val MethodEnd: Value = Value("}METHOD")
-  val VariableDefinition: Value = Value("VAR_DEF")
-  val DoWhile: Value = Value("DO-WHILE")
-  val DoWhileEnd: Value = Value("END-DO-WHILE")
-  val DoBodyBegin: Value = Value("DO{")
-  val DoBodyEnd: Value = Value("}DO")
-  val While: Value = Value("WHILE")
-  val WhileBodyBegin: Value = Value("WHILE{")
-  val WhileBodyEnd: Value = Value("}WHILE")
-  val For: Value = Value("FOR")
-  val ForBodyBegin: Value = Value("FOR{")
-  val ForBodyEnd: Value = Value("}FOR")
-  val CaseStatement: Value = Value("CASE")
-  val CaseBegin: Value = Value("CASE{")
-  val CaseEnd: Value = Value("}CASE")
-  val TryBegin: Value = Value("TRY{")
-  val CatchBegin: Value = Value("CATCH{")
-  val CatchEnd: Value = Value("}CATCH")
-  val Finally: Value = Value("FINALLY")
-  val If: Value = Value("IF")
-  val IfBegin: Value = Value("IF{")
-  val IfEnd: Value = Value("}IF")
-  val Else: Value = Value("ELSE")
-  val ElseBegin: Value = Value("ELSE{")
-  val ElseEnd: Value = Value("}ELSE")
-  val Return: Value = Value("RETURN")
-  val Throw: Value = Value("THROW")
-  val NewCreationBegin: Value = Value("NEW{")
-  val NewCreationEnd: Value = Value("}NEW")
-  val Apply: Value = Value("APPLY")
-  val Assign: Value = Value("ASSIGN")
-  val TraitBegin: Value = Value("TRAIT{")
-  val TraitEnd: Value = Value("}TRAIT")
-  val ConstructorBegin: Value = Value("CONSTR{")
-  val ConstructorEnd: Value = Value("}CONSTR")
-  val MatchBegin: Value = Value("MATCH{")
-  val MatchEnd: Value = Value("}MATCH")
-  val Guard: Value = Value("GUARD")
-  val ObjectBegin: Value = Value("OBJECT{")
-  val ObjectEnd: Value = Value("}OBJECT")
-  val Macro: Value = Value("MACRO")
-  val MacroBegin: Value = Value("MACRO{")
-  val MacroEnd: Value = Value("}MACRO")
-  val Type: Value = Value("TYPE")
-
-  val FunctionBegin: Value = Value("FUNC{")
-  val FunctionEnd: Value = Value("}FUNC")
-  val PartialFunctionBegin: Value = Value("PFUNC{")
-  val PartialFunctionEnd: Value = Value("}PFUNC")
-
-  val Yield: Value = Value("YIELD")
-
-  val Parameter: Value = Value("PARAM")
-  val Argument: Value = Value("ARG")
-  val NewObject: Value = Value("NEW()")
-  val SelfType: Value = Value("SELF")
-  val TypeParameter: Value = Value("T_PARAM")
-  val TypeArgument: Value = Value("T_ARG")
-  val BlockStart: Value = Value("{")
-  val BlockEnd: Value = Value("}")
-  val EnumGenerator: Value = Value("ENUMERATE")
-  val Member: Value = Value("MEMBER")
-
-  def numberOfTokens() : Int = maxId
-}
-

languages/scala/src/main/scala/de/jplag/scala/ScalaTokenType.java

@@ -0,0 +1,84 @@
+package de.jplag.scala;
+
+import de.jplag.TokenType;
+
+public enum ScalaTokenType implements TokenType {
+    PACKAGE("PACKAGE"),
+    IMPORT("IMPORT"),
+    CLASS_BEGIN("CLASS{"),
+    CLASS_END("}CLASS"),
+    METHOD_DEF("METHOD"),
+    METHOD_BEGIN("METHOD{"),
+    METHOD_END("}METHOD"),
+    VARIABLE_DEFINITION("VAR_DEF"),
+    DO_WHILE("DO-WHILE"),
+    DO_WHILE_END("END-DO-WHILE"),
+    DO_BODY_BEGIN("DO{"),
+    DO_BODY_END("}DO"),
+    WHILE("WHILE"),
+    WHILE_BODY_BEGIN("WHILE{"),
+    WHILE_BODY_END("}WHILE"),
+    FOR("FOR"),
+    FOR_BODY_BEGIN("FOR{"),
+    FOR_BODY_END("}FOR"),
+    CASE_STATEMENT("CASE"),
+    CASE_BEGIN("CASE{"),
+    CASE_END("}CASE"),
+    TRY_BEGIN("TRY{"),
+    CATCH_BEGIN("CATCH{"),
+    CATCH_END("}CATCH"),
+    FINALLY("FINALLY"),
+    IF("IF"),
+    IF_BEGIN("IF{"),
+    IF_END("}IF"),
+    ELSE("ELSE"),
+    ELSE_BEGIN("ELSE{"),
+    ELSE_END("}ELSE"),
+    RETURN("RETURN"),
+    THROW("THROW"),
+    NEW_CREATION_BEGIN("NEW{"),
+    NEW_CREATION_END("}NEW"),
+    APPLY("APPLY"),
+    ASSIGN("ASSIGN"),
+    TRAIT_BEGIN("TRAIT{"),
+    TRAIT_END("}TRAIT"),
+    CONSTRUCTOR_BEGIN("CONSTR{"),
+    CONSTRUCTOR_END("}CONSTR"),
+    MATCH_BEGIN("MATCH{"),
+    MATCH_END("}MATCH"),
+    GUARD("GUARD"),
+    OBJECT_BEGIN("OBJECT{"),
+    OBJECT_END("}OBJECT"),
+    MACRO("MACRO"),
+    MACRO_BEGIN("MACRO{"),
+    MACRO_END("}MACRO"),
+    TYPE("TYPE"),
+
+    FUNCTION_BEGIN("FUNC{"),
+    FUNCTION_END("}FUNC"),
+    PARTIAL_FUNCTION_BEGIN("PFUNC{"),
+    PARTIAL_FUNCTION_END("}PFUNC"),
+
+    YIELD("YIELD"),
+
+    PARAMETER("PARAM"),
+    ARGUMENT("ARG"),
+    NEW_OBJECT("NEW(),"),
+    SELF_TYPE("SELF"),
+    TYPE_PARAMETER("T_PARAM"),
+    TYPE_ARGUMENT("T_ARG"),
+    BLOCK_START("{"),
+    BLOCK_END("}"),
+    ENUM_GENERATOR("ENUMERATE"),
+    MEMBER("MEMBER");
+
+    private final String description;
+
+    public String getDescription() {
+        return description;
+    }
+
+    ScalaTokenType(String description) {
+        this.description = description;
+    }
+}

languages/scala/src/test/java/de/jplag/scala/ScalaFrontendTest.java

@@ -1,5 +1,6 @@
 package de.jplag.scala;
 
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
 import java.io.File;
@@ -7,6 +8,7 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -16,8 +18,8 @@ import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import de.jplag.SharedTokenType;
 import de.jplag.Token;
-import de.jplag.TokenConstants;
 import de.jplag.TokenPrinter;
 
 class ScalaFrontendTest {
@@ -41,8 +43,6 @@ class ScalaFrontendTest {
      * Regular expression that describes lines containing the end of a multiline comment and no more code after that.
      */
     private static final String DELIMITED_COMMENT_END = ".*\\*/\\s*$";
-    private static final String EMPTY_STRING = "";
-    private static final int NOT_SET = -1;
     private static final double EPSILON = 1E-6;
 
     private final Logger logger = LoggerFactory.getLogger("Scala frontend test");
@@ -70,22 +70,6 @@ class ScalaFrontendTest {
         }
     }
 
-    /**
-     * Confirms that every type of ScalaToken has a Sting representation associated to it.
-     */
-    @Test
-    void testTokenToString() {
-        var missingTokens = IntStream.range(0, ScalaTokenConstants.numberOfTokens())
-                .mapToObj(type -> new ScalaToken(type, EMPTY_STRING, NOT_SET, NOT_SET, NOT_SET))
-                .filter(token -> token.type2string().contains("UNKNOWN")).toList();
-
-        if (!missingTokens.isEmpty()) {
-            var typeList = missingTokens.stream().map(Token::getType).map(Object::toString).collect(Collectors.joining(", "));
-            fail("Found token types with no string representation: %s".formatted(typeList));
-        }
-
-    }
-
     /**
      * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
      * @param fileName a code sample file name
@@ -100,7 +84,7 @@ class ScalaFrontendTest {
             // All lines that contain code
             var codeLines = new ArrayList<>(getCodeLines(lines));
             // All lines that contain token
-            var tokenLines = tokens.stream().map(Token::getLine).filter(line -> line != Token.NO_VALUE).distinct().toList();
+            var tokenLines = tokens.stream().map(Token::getLine).distinct().toList();
 
             // Keep only lines that have no tokens
             codeLines.removeAll(tokenLines);
@@ -133,7 +117,7 @@ class ScalaFrontendTest {
             boolean insideComment = false;
         };
 
-        var codeLines = IntStream.rangeClosed(1, lines.size()).sequential().filter(idx -> {
+        return IntStream.rangeClosed(1, lines.size()).sequential().filter(idx -> {
             String line = lines.get(idx - 1);
             if (line.matches(EMPTY_OR_SINGLE_LINE_COMMENT)) {
                 return false;
@@ -149,9 +133,6 @@ class ScalaFrontendTest {
             }
             return true;
         }).boxed().toList();
-
-        return codeLines;
-
     }
 
     /**
@@ -160,21 +141,12 @@ class ScalaFrontendTest {
      * @param fileName The file name of the complete code example
      */
     private void testTokenCoverage(List<Token> tokens, String fileName) {
-        var foundTokens = tokens.stream().parallel().map(Token::getType).distinct().toList();
-        var allTokens = IntStream.range(0, ScalaTokenConstants.numberOfTokens()).boxed().toList();
-        allTokens = new ArrayList<>(allTokens);
-
-        // Only non-found tokens are left
-        allTokens.removeAll(foundTokens);
-        // Exclude SEPARATOR_TOKEN, as it does not occur
-        allTokens.remove((Integer) (TokenConstants.SEPARATOR_TOKEN));
-
-        if (!allTokens.isEmpty()) {
-            var notFoundTypes = allTokens.stream().map(ScalaTokenConstants::apply).toList();
-            fail("Some %d token types were not found in the complete code example '%s':\n%s".formatted(notFoundTypes.size(), fileName,
-                    notFoundTypes));
-        }
-
+        var annotatedTokens = tokens.stream().map(Token::getType).collect(Collectors.toSet());
+        assertTrue(annotatedTokens.contains(SharedTokenType.FILE_END));
+        var annotatedScalaTokens = annotatedTokens.stream().filter(ScalaTokenType.class::isInstance).collect(Collectors.toSet());
+        var allScalaTokens = ScalaTokenType.values();
+        var missingScalaTokens = Arrays.stream(allScalaTokens).filter(token -> !annotatedScalaTokens.contains(token)).toList();
+        assertTrue(missingScalaTokens.isEmpty(), "The following scala tokens are missing in the code example '%s':\n".formatted(fileName)
+                + String.join("\n", missingScalaTokens.stream().map(ScalaTokenType::getDescription).toList()));
     }
-
-}
+}

languages/scheme/src/main/java/de/jplag/scheme/Parser.java

@@ -5,12 +5,13 @@ import java.util.ArrayList;
 import java.util.List;
 
 import de.jplag.AbstractParser;
-import de.jplag.TokenConstants;
+import de.jplag.Token;
+import de.jplag.TokenType;
 
 public class Parser extends AbstractParser {
     private String currentFile;
 
-    private List<de.jplag.Token> tokens;
+    private List<Token> tokens;
 
     /**
      * Creates the parser.
@@ -19,22 +20,23 @@ public class Parser extends AbstractParser {
         super();
     }
 
-    public List<de.jplag.Token> parse(File directory, String[] files) {
+    public List<Token> parse(File directory, String[] files) {
         tokens = new ArrayList<>();
         errors = 0;
         for (String file : files) {
             currentFile = file;
             logger.trace("Parsing file {}", file);
-            if (!SchemeParser.parseFile(directory, file, null, this))
+            if (!SchemeParser.parseFile(directory, file, null, this)) {
                 errors++;
-            tokens.add(new SchemeToken(TokenConstants.FILE_END, currentFile));
+            }
+            tokens.add(Token.fileEnd(currentFile));
         }
         return tokens;
     }
 
-    public void add(int type, Token token) {
+    public void add(TokenType type, de.jplag.scheme.Token token) {
         int length = token.endColumn - token.beginColumn + 1;
-        tokens.add(new SchemeToken(type, currentFile, token.beginLine, token.endLine, length));
+        tokens.add(new Token(type, currentFile, token.beginLine, token.endLine, length));
     }
 
 }

languages/scheme/src/main/java/de/jplag/scheme/SchemeToken.java

@@ -1,62 +0,0 @@
-package de.jplag.scheme;
-
-import de.jplag.Token;
-
-public class SchemeToken extends Token implements SchemeTokenConstants {
-
-    public SchemeToken(int type, String file, int line, int column, int length) {
-        super(type, file, line, column, length);
-    }
-
-    public SchemeToken(int type, String file) {
-        super(type, file, 1, 0, 0);
-    }
-
-    protected String type2string() {
-        return switch (type) {
-            case SchemeTokenConstants.FILE_END -> "EOF";
-            case S_BOOL -> "BOOL    ";
-            case S_NUMBER -> "NUMBER  ";
-            case S_CHAR -> "CHAR    ";
-            case S_STRING -> "STRING  ";
-            case S_ID -> "ID      ";
-            case S_LIST_BEGIN -> "(LIST   ";
-            case S_LIST_END -> "LIST)   ";
-            case S_VECTOR_BEGIN -> "(VECTOR ";
-            case S_VECTOR_END -> "VECTOR) ";
-            case S_LITERAL -> "LITERAL ";
-            case S_QUOT_BEGIN -> "(QUOT   ";
-            case S_QUOT_END -> "QUOT)   ";
-            case S_CALL -> "CALL    ";
-            case S_LAMBDA_BEGIN -> "(LAMBDA ";
-            case S_LAMBDA_END -> "LAMBDA) ";
-            case S_FORMAL_BEGIN -> "(FORMAL ";
-            case S_FORMAL_END -> "FORMAL) ";
-            case S_BODY_BEGIN -> "(BODY   ";
-            case S_BODY_END -> "BODY)   ";
-            case S_IF_BEGIN -> "(IF     ";
-            case S_IF_END -> "IF)     ";
-            case S_ALTERN -> "ALTERN  ";
-            case S_ASSIGN_BEGIN -> "(ASSIGN ";
-            case S_ASSIGN_END -> "ASSIGN) ";
-            case S_COND_BEGIN -> "(COND   ";
-            case S_COND_END -> "COND)   ";
-            case S_ELSE -> "ELSE    ";
-            case S_CASE_BEGIN -> "(CASE   ";
-            case S_CASE_END -> "CASE)   ";
-            case S_DO_BEGIN -> "(DO     ";
-            case S_DO_END -> "DO)     ";
-            case S_COMMAND -> "COMMAND ";
-            case S_DEF_BEGIN -> "(DEF    ";
-            case S_DEF_END -> "DEF)    ";
-            case S_BEGIN -> "(       ";
-            case S_END -> ")       ";
-            case S_AND -> "AND     ";
-            case S_OR -> "OR      ";
-            case S_LET -> "LET     ";
-            case S_DELAY -> "DELAY   ";
-            case S_VAR -> "VAR     ";
-            default -> "<?????> ";
-        };
-    }
-}

languages/scheme/src/main/java/de/jplag/scheme/SchemeTokenConstants.java

@@ -1,50 +0,0 @@
-package de.jplag.scheme;
-
-import de.jplag.TokenConstants;
-
-public interface SchemeTokenConstants extends TokenConstants {
-
-    int S_BOOL = 2;
-    int S_NUMBER = 3;
-    int S_CHAR = 4;
-    int S_STRING = 5;
-    int S_ID = 6;
-    int S_LIST_BEGIN = 7;
-    int S_LIST_END = 8;
-    int S_VECTOR_BEGIN = 9;
-    int S_VECTOR_END = 10;
-    int S_LITERAL = 11;
-    int S_QUOT_BEGIN = 12;
-    int S_QUOT_END = 13;
-    int S_CALL = 14;
-    int S_LAMBDA_BEGIN = 15;
-    int S_LAMBDA_END = 16;
-    int S_FORMAL_BEGIN = 17;
-    int S_FORMAL_END = 18;
-    int S_BODY_BEGIN = 19;
-    int S_BODY_END = 20;
-    int S_IF_BEGIN = 21;
-    int S_IF_END = 22;
-    int S_ALTERN = 23;
-    int S_ASSIGN_BEGIN = 24;
-    int S_ASSIGN_END = 25;
-    int S_COND_BEGIN = 26;
-    int S_COND_END = 27;
-    int S_ELSE = 28;
-    int S_CASE_BEGIN = 29;
-    int S_CASE_END = 30;
-    int S_DO_BEGIN = 31;
-    int S_DO_END = 32;
-    int S_COMMAND = 33;
-    int S_DEF_BEGIN = 34;
-    int S_DEF_END = 35;
-    int S_BEGIN = 36;
-    int S_END = 37;
-    int S_AND = 38;
-    int S_OR = 39;
-    int S_LET = 40;
-    int S_DELAY = 41;
-    int S_VAR = 42;
-
-    int NUM_DIFF_TOKENS = 43;
-}

languages/scheme/src/main/java/de/jplag/scheme/SchemeTokenType.java

@@ -0,0 +1,57 @@
+package de.jplag.scheme;
+
+import de.jplag.TokenType;
+
+public enum SchemeTokenType implements TokenType {
+    S_BOOL("BOOL"),
+    S_NUMBER("NUMBER"),
+    S_CHAR("CHAR"),
+    S_STRING("STRING"),
+    S_ID("ID"),
+    S_LIST_BEGIN("(LIST"),
+    S_LIST_END("LIST)"),
+    S_VECTOR_BEGIN("(VECTOR"),
+    S_VECTOR_END("VECTOR)"),
+    S_LITERAL("LITERAL"),
+    S_QUOT_BEGIN("(QUOT"),
+    S_QUOT_END("QUOT)"),
+    S_CALL("CALL"),
+    S_LAMBDA_BEGIN("(LAMBDA"),
+    S_LAMBDA_END("LAMBDA)"),
+    S_FORMAL_BEGIN("(FORMAL"),
+    S_FORMAL_END("FORMAL)"),
+    S_BODY_BEGIN("(BODY"),
+    S_BODY_END("BODY)"),
+    S_IF_BEGIN("(IF"),
+    S_IF_END("IF)"),
+    S_ALTERN("ALTERN"),
+    S_ASSIGN_BEGIN("(ASSIGN"),
+    S_ASSIGN_END("ASSIGN)"),
+    S_COND_BEGIN("(COND"),
+    S_COND_END("COND)"),
+    S_ELSE("ELSE"),
+    S_CASE_BEGIN("(CASE"),
+    S_CASE_END("CASE)"),
+    S_DO_BEGIN("(DO"),
+    S_DO_END("DO)"),
+    S_COMMAND("COMMAND"),
+    S_DEF_BEGIN("(DEF"),
+    S_DEF_END("DEF)"),
+    S_BEGIN("("),
+    S_END(")"),
+    S_AND("AND"),
+    S_OR("OR"),
+    S_LET("LET"),
+    S_DELAY("DELAY"),
+    S_VAR("VAR");
+
+    private final String description;
+
+    public String getDescription() {
+        return this.description;
+    }
+
+    SchemeTokenType(String description) {
+        this.description = description;
+    }
+}

languages/scheme/src/main/javacc/Scheme.jj

@@ -52,11 +52,13 @@ options
 PARSER_BEGIN(SchemeParser)
 package de.jplag.scheme;
 
+import static de.jplag.scheme.SchemeTokenType.*;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 
-public class SchemeParser implements SchemeTokenConstants {
+public class SchemeParser {
     /* used for context in the template production rule */
     private static int templateParam;
     private Parser parser2;

languages/text/src/main/java/de/jplag/text/ParserAdapter.java

@@ -5,14 +5,11 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Properties;
 
 import de.jplag.AbstractParser;
 import de.jplag.Token;
-import de.jplag.TokenConstants;
 
 import edu.stanford.nlp.ling.CoreLabel;
 import edu.stanford.nlp.pipeline.CoreDocument;
@@ -24,9 +21,7 @@ public class ParserAdapter extends AbstractParser {
     private static final char CR = '\r';
     private static final String ANNOTATORS_KEY = "annotators";
     private static final String ANNOTATORS_VALUE = "tokenize";
-    private final Map<String, Integer> tokenTypes = new HashMap<>();
     private final StanfordCoreNLP pipeline;
-    private int tokenTypeIndex = 2; // 0 is FILE_END token, 1 is SEPARATOR_TOKEN, so start at 2.
 
     private List<Token> tokens;
     private String currentFile;
@@ -50,7 +45,7 @@ public class ParserAdapter extends AbstractParser {
             if (!parseFile(directory, file)) {
                 errors++;
             }
-            tokens.add(new TextToken(TokenConstants.FILE_END, file));
+            tokens.add(Token.fileEnd(file));
         }
         return tokens;
     }
@@ -105,10 +100,9 @@ public class ParserAdapter extends AbstractParser {
 
     private void addToken(CoreLabel label) {
         String text = label.originalText();
-        int type = getTokenType(text);
         int column = label.beginPosition() - currentLineBreakIndex;
         int length = label.endPosition() - label.beginPosition();
-        tokens.add(new TextToken(text, type, currentFile, currentLine, column, length));
+        tokens.add(new Token(new TextTokenType(text), currentFile, currentLine, column, length));
     }
 
     private String readFile(Path filePath) {
@@ -119,15 +113,4 @@ public class ParserAdapter extends AbstractParser {
             return null;
         }
     }
-
-    private int getTokenType(String text) {
-        text = text.toLowerCase();
-        tokenTypes.computeIfAbsent(text, it -> {
-            if (tokenTypeIndex == Integer.MAX_VALUE) {
-                throw new IllegalStateException("Too many token types, should not happen!");
-            }
-            return ++tokenTypeIndex;
-        });
-        return tokenTypes.get(text);
-    }
 }

languages/text/src/main/java/de/jplag/text/TextToken.java

@@ -1,28 +0,0 @@
-package de.jplag.text;
-
-import de.jplag.Token;
-
-public class TextToken extends Token {
-
-    private static final String NO_TEXT = "<unknown>";
-    private final String text;
-
-    public TextToken(int type, String file) {
-        super(type, file, -1, -1, -1);
-        this.text = NO_TEXT;
-    }
-
-    public TextToken(String text, int type, String file, int line, int column, int length) {
-        super(type, file, line, column, length);
-        this.text = text.toLowerCase();
-    }
-
-    public String getText() {
-        return this.text;
-    }
-
-    @Override
-    protected String type2string() {
-        return getText();
-    }
-}

languages/text/src/main/java/de/jplag/text/TextTokenType.java

@@ -0,0 +1,13 @@
+package de.jplag.text;
+
+import de.jplag.TokenType;
+
+public record TextTokenType(String description) implements TokenType {
+    public TextTokenType(String description) {
+        this.description = description.toLowerCase();
+    }
+
+    public String getDescription() {
+        return this.description;
+    }
+}

languages/text/src/test/java/jplag/text/TextFrontendTest.java

@@ -7,9 +7,8 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -21,6 +20,7 @@ import org.slf4j.LoggerFactory;
 
 import de.jplag.Token;
 import de.jplag.TokenPrinter;
+import de.jplag.TokenType;
 import de.jplag.text.Language;
 
 class TextFrontendTest {
@@ -46,12 +46,9 @@ class TextFrontendTest {
         List<Token> result = frontend.parse(baseDirectory, input);
         logger.info(TokenPrinter.printTokens(result, baseDirectory));
 
-        // Compare parsed tokens:
-        Map<Integer, Token> tokenTypes = new HashMap<>();
-        result.forEach(it -> tokenTypes.put(it.getType(), it));
-
-        assertEquals(283, result.size());
-        assertEquals(158, tokenTypes.values().size());
+        List<TokenType> tokenTypes = result.stream().map(Token::getType).toList();
+        assertEquals(283, tokenTypes.size());
+        assertEquals(158, new HashSet<>(tokenTypes).size());
     }
 
     @ParameterizedTest

pom.xml

@@ -81,7 +81,7 @@
         <antlr2.version>2.7.7</antlr2.version>
         <antlr4.version>4.11.1</antlr4.version>
         <emf.version>2.28.0</emf.version>
-        <emf.ecore.version>2.25.0</emf.ecore.version>
+        <emf.ecore.version>2.26.0</emf.ecore.version>
         <emf.ecore.xmi.version>2.17.0</emf.ecore.xmi.version>
 
         <!-- The Revision of JPlag -->
@@ -166,7 +166,7 @@
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
-            <version>4.7.0</version>
+            <version>4.8.0</version>
             <scope>test</scope>
         </dependency>
         <dependency>