adapt Swift language frontend to language interface overhauls

languages/swift/src/main/java/de/jplag/swift/JPlagSwiftListener.java

@@ -1,6 +1,6 @@
 package de.jplag.swift;
 
-import static de.jplag.swift.SwiftTokenConstants.*;
+import static de.jplag.swift.SwiftTokenType.*;
 
 import org.antlr.v4.runtime.Token;
 
@@ -19,7 +19,7 @@ public class JPlagSwiftListener extends Swift5ParserBaseListener {
      * @param tokenType the custom token type that occurred.
      * @param token the corresponding grammar's token
      */
-    private void transformToken(int tokenType, Token token) {
+    private void transformToken(SwiftTokenType tokenType, Token token) {
         parserAdapter.addToken(tokenType, token.getLine(), token.getCharPositionInLine() + 1, token.getText().length());
     }
 
@@ -30,7 +30,7 @@ public class JPlagSwiftListener extends Swift5ParserBaseListener {
      * @param start the first Token of the context
      * @param end the last Token of the context
      */
-    private void transformToken(int tokenType, Token start, Token end) {
+    private void transformToken(SwiftTokenType tokenType, Token start, Token end) {
         parserAdapter.addToken(tokenType, start.getLine(), start.getCharPositionInLine() + 1, end.getStopIndex() - start.getStartIndex() + 1);
     }
 

languages/swift/src/main/java/de/jplag/swift/Language.java

@@ -1,8 +1,9 @@
 package de.jplag.swift;
 
 import java.io.File;
+import java.util.List;
 
-import de.jplag.TokenList;
+import de.jplag.Token;
 
 /**
  * This represents the Swift language as a language supported by JPlag.
@@ -10,7 +11,7 @@ import de.jplag.TokenList;
 public class Language implements de.jplag.Language {
 
     private static final String NAME = "Swift Parser";
-    private static final String SHORT_NAME = "Swift";
+    private static final String IDENTIFIER = "Swift";
     private static final int DEFAULT_MIN_TOKEN_MATCH = 8;
     private static final String[] FILE_EXTENSIONS = {".swift"};
     private final SwiftParserAdapter parserAdapter;
@@ -30,8 +31,8 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public String getShortName() {
-        return SHORT_NAME;
+    public String getIdentifier() {
+        return IDENTIFIER;
     }
 
     @Override
@@ -40,7 +41,7 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File directory, String[] files) {
+    public List<Token> parse(File directory, String[] files) {
         return parserAdapter.parse(directory, files);
     }
 

languages/swift/src/main/java/de/jplag/swift/SwiftParserAdapter.java

@@ -3,6 +3,8 @@ package de.jplag.swift;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
@@ -12,8 +14,6 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker;
 
 import de.jplag.AbstractParser;
 import de.jplag.Token;
-import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.swift.grammar.Swift5Lexer;
 import de.jplag.swift.grammar.Swift5Parser;
 
@@ -21,7 +21,7 @@ public class SwiftParserAdapter extends AbstractParser {
 
     public static final int NOT_SET = -1;
     private String currentFile;
-    private TokenList tokens;
+    private List<Token> tokens;
 
     /**
      * Creates the SwiftParserAdapter
@@ -36,13 +36,13 @@ public class SwiftParserAdapter extends AbstractParser {
      * @param fileNames the file names of the files.
      * @return a list containing all tokens of all files.
      */
-    public TokenList parse(File directory, String[] fileNames) {
-        tokens = new TokenList();
+    public List<Token> parse(File directory, String[] fileNames) {
+        tokens = new ArrayList<>();
         for (String file : fileNames) {
             if (!parseFile(directory, file)) {
                 errors++;
             }
-            tokens.addToken(new SwiftToken(TokenConstants.FILE_END, file, NOT_SET, NOT_SET, NOT_SET));
+            tokens.add(Token.fileEnd(file));
         }
         return tokens;
     }
@@ -78,7 +78,7 @@ public class SwiftParserAdapter extends AbstractParser {
      * @param column the start column of the Token in the line
      * @param length the length of the Token
      */
-    /* package-private */ void addToken(int tokenType, int line, int column, int length) {
-        tokens.addToken(new SwiftToken(tokenType, currentFile, line, column, length));
+    /* package-private */ void addToken(SwiftTokenType tokenType, int line, int column, int length) {
+        tokens.add(new Token(tokenType, currentFile, line, column, length));
     }
 }

languages/swift/src/main/java/de/jplag/swift/SwiftToken.java

@@ -1,64 +0,0 @@
-package de.jplag.swift;
-
-import static de.jplag.swift.SwiftTokenConstants.*;
-
-import de.jplag.Token;
-
-public class SwiftToken extends Token {
-
-    public SwiftToken(int type, String file, int line, int column, int length) {
-        super(type, file, line, column, length);
-    }
-
-    @Override
-    public String type2string() {
-        return switch (type) {
-            case FILE_END -> "<EOF>";
-            case SEPARATOR_TOKEN -> "-----";
-            case IMPORT -> "IMPORT";
-            case CLASS_DECLARATION -> "CLASS";
-            case STRUCT_DECLARATION -> "STRUCT";
-            case ENUM_DECLARATION -> "ENUM";
-            case PROTOCOL_DECLARATION -> "PROTOCOL";
-            case CLASS_BODY_BEGIN -> "CLASS{";
-            case CLASS_BODY_END -> "}CLASS";
-            case STRUCT_BODY_BEGIN -> "STRUCT{";
-            case STRUCT_BODY_END -> "}STRUCT";
-            case ENUM_BODY_BEGIN -> "ENUM{";
-            case ENUM_BODY_END -> "}ENUM";
-            case ENUM_LITERAL -> "ENUM CASE";
-            case PROTOCOL_BODY_BEGIN -> "PROTOCOL{";
-            case PROTOCOL_BODY_END -> "}PROTOCOL";
-            case PROPERTY_DECLARATION -> "PROPERTY";
-            case PROPERTY_ACCESSOR_BEGIN -> "ACCESSOR{";
-            case PROPERTY_ACCESSOR_END -> "}ACCESSOR";
-            case FUNCTION -> "FUNCTION";
-            case FUNCTION_PARAMETER -> "PARAMETER";
-            case FUNCTION_BODY_BEGIN -> "FUNCTION{";
-            case FUNCTION_BODY_END -> "}FUNCTION";
-            case CLOSURE_BODY_BEGIN -> "CLOSURE{";
-            case CLOSURE_BODY_END -> "}CLOSURE";
-            case FOR_BODY_BEGIN -> "FOR{";
-            case FOR_BODY_END -> "}FOR";
-            case IF_BODY_BEGIN -> "IF{";
-            case IF_BODY_END -> "}IF";
-            case SWITCH_BODY_BEGIN -> "SWITCH{";
-            case SWITCH_BODY_END -> "}SWITCH";
-            case SWITCH_CASE -> "CASE";
-            case WHILE_BODY_BEGIN -> "WHILE{";
-            case WHILE_BODY_END -> "}WHILE";
-            case REPEAT_WHILE_BODY_BEGIN -> "REPEAT{";
-            case REPEAT_WHILE_BODY_END -> "}REPEAT";
-            case DEFER_BODY_BEGIN -> "DEFER{";
-            case DEFER_BODY_END -> "}DEFER";
-            case THROW -> "THROW";
-            case RETURN -> "RETURN";
-            case CONTINUE -> "CONTINUE";
-            case BREAK -> "BREAK";
-            case FALLTHROUGH -> "FALLTHROUGH";
-            case ASSIGNMENT -> "ASSIGN";
-            case FUNCTION_CALL -> "CALL";
-            default -> "<UNKNOWN%d>".formatted(type);
-        };
-    }
-}

languages/swift/src/main/java/de/jplag/swift/SwiftTokenConstants.java

@@ -1,67 +0,0 @@
-package de.jplag.swift;
-
-import de.jplag.TokenConstants;
-
-public interface SwiftTokenConstants extends TokenConstants {
-
-    int IMPORT = 2;
-
-    int CLASS_DECLARATION = 3;
-    int STRUCT_DECLARATION = 4;
-    int ENUM_DECLARATION = 5;
-    int PROTOCOL_DECLARATION = 6;
-
-    int CLASS_BODY_BEGIN = 7;
-    int CLASS_BODY_END = 8;
-
-    int STRUCT_BODY_BEGIN = 9;
-    int STRUCT_BODY_END = 10;
-
-    int ENUM_BODY_BEGIN = 11;
-    int ENUM_BODY_END = 12;
-    int ENUM_LITERAL = 13;
-
-    int PROTOCOL_BODY_BEGIN = 14;
-    int PROTOCOL_BODY_END = 15;
-
-    int PROPERTY_DECLARATION = 16;
-    int PROPERTY_ACCESSOR_BEGIN = 17;
-    int PROPERTY_ACCESSOR_END = 18;
-
-    int FUNCTION = 19;
-    int FUNCTION_PARAMETER = 20;
-    int FUNCTION_BODY_BEGIN = 21;
-    int FUNCTION_BODY_END = 22;
-
-    int CLOSURE_BODY_BEGIN = 23;
-    int CLOSURE_BODY_END = 24;
-
-    int FOR_BODY_BEGIN = 25;
-    int FOR_BODY_END = 26;
-
-    int IF_BODY_BEGIN = 27;
-    int IF_BODY_END = 28;
-
-    int SWITCH_BODY_BEGIN = 29;
-    int SWITCH_BODY_END = 30;
-    int SWITCH_CASE = 31;
-
-    int WHILE_BODY_BEGIN = 32;
-    int WHILE_BODY_END = 33;
-
-    int REPEAT_WHILE_BODY_BEGIN = 34;
-    int REPEAT_WHILE_BODY_END = 35;
-
-    int DEFER_BODY_BEGIN = 36;
-    int DEFER_BODY_END = 37;
-    int THROW = 38;
-    int RETURN = 39;
-    int CONTINUE = 40;
-    int BREAK = 41;
-    int FALLTHROUGH = 42;
-
-    int ASSIGNMENT = 43;
-    int FUNCTION_CALL = 44;
-
-    int NUMBER_DIFF_TOKENS = 45;
-}

languages/swift/src/main/java/de/jplag/swift/SwiftTokenType.java

@@ -0,0 +1,59 @@
+package de.jplag.swift;
+
+import de.jplag.TokenType;
+
+public enum SwiftTokenType implements TokenType {
+    IMPORT("IMPORT"),
+    CLASS_DECLARATION("CLASS"),
+    STRUCT_DECLARATION("STRUCT"),
+    ENUM_DECLARATION("ENUM"),
+    PROTOCOL_DECLARATION("PROTOCOL"),
+    CLASS_BODY_BEGIN("CLASS{"),
+    CLASS_BODY_END("}CLASS"),
+    STRUCT_BODY_BEGIN("STRUCT{"),
+    STRUCT_BODY_END("}STRUCT"),
+    ENUM_BODY_BEGIN("ENUM{"),
+    ENUM_BODY_END("}ENUM"),
+    ENUM_LITERAL("ENUM CASE"),
+    PROTOCOL_BODY_BEGIN("PROTOCOL{"),
+    PROTOCOL_BODY_END("}PROTOCOL"),
+    PROPERTY_DECLARATION("PROPERTY"),
+    PROPERTY_ACCESSOR_BEGIN("ACCESSOR{"),
+    PROPERTY_ACCESSOR_END("}ACCESSOR"),
+    FUNCTION("FUNCTION"),
+    FUNCTION_PARAMETER("PARAMETER"),
+    FUNCTION_BODY_BEGIN("FUNCTION{"),
+    FUNCTION_BODY_END("}FUNCTION"),
+    CLOSURE_BODY_BEGIN("CLOSURE{"),
+    CLOSURE_BODY_END("}CLOSURE"),
+    FOR_BODY_BEGIN("FOR{"),
+    FOR_BODY_END("}FOR"),
+    IF_BODY_BEGIN("IF{"),
+    IF_BODY_END("}IF"),
+    SWITCH_BODY_BEGIN("SWITCH{"),
+    SWITCH_BODY_END("}SWITCH"),
+    SWITCH_CASE("CASE"),
+    WHILE_BODY_BEGIN("WHILE{"),
+    WHILE_BODY_END("}WHILE"),
+    REPEAT_WHILE_BODY_BEGIN("REPEAT{"),
+    REPEAT_WHILE_BODY_END("}REPEAT"),
+    DEFER_BODY_BEGIN("DEFER{"),
+    DEFER_BODY_END("}DEFER"),
+    THROW("THROW"),
+    RETURN("RETURN"),
+    CONTINUE("CONTINUE"),
+    BREAK("BREAK"),
+    FALLTHROUGH("FALLTHROUGH"),
+    ASSIGNMENT("ASSIGN"),
+    FUNCTION_CALL("CALL");
+
+    private final String description;
+
+    public String getDescription() {
+        return description;
+    }
+
+    SwiftTokenType(String description) {
+        this.description = description;
+    }
+}

languages/swift/src/test/java/de/jplag/swift/SwiftFrontendTest.java

@@ -1,25 +1,26 @@
 package de.jplag.swift;
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
-import java.util.stream.StreamSupport;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import de.jplag.SharedTokenType;
 import de.jplag.Token;
-import de.jplag.TokenConstants;
-import de.jplag.TokenList;
 import de.jplag.TokenPrinter;
 
 class SwiftFrontendTest {
@@ -48,8 +49,6 @@ class SwiftFrontendTest {
      * Regular expression that describes lines containing the end of a multiline comment and no more code after that.
      */
     private static final String DELIMITED_COMMENT_END = ".*\\*/\\s*$";
-    private static final String NOT_SET_STRING = "";
-    private static final int NOT_SET = -1;
 
     private final Logger logger = LoggerFactory.getLogger("Swift frontend test");
     private final String[] testFiles = new String[] {COMPLETE_TEST_FILE};
@@ -64,8 +63,8 @@ class SwiftFrontendTest {
     @Test
     void parseTestFiles() {
         for (String fileName : testFiles) {
-            TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
-            String output = TokenPrinter.printTokens(tokens, testFileLocation, List.of(fileName));
+            List<Token> tokens = language.parse(testFileLocation, new String[] {fileName});
+            String output = TokenPrinter.printTokens(tokens, testFileLocation, Optional.empty());
             logger.info(output);
 
             testSourceCoverage(fileName, tokens);
@@ -76,28 +75,12 @@ class SwiftFrontendTest {
         }
     }
 
-    /**
-     * Confirms that every type of SwiftToken has a Sting representation associated to it.
-     */
-    @Test
-    void testTokenToString() {
-        var missingTokens = IntStream.range(0, SwiftTokenConstants.NUMBER_DIFF_TOKENS)
-                .mapToObj(type -> new SwiftToken(type, NOT_SET_STRING, NOT_SET, NOT_SET, NOT_SET))
-                .filter(token -> token.type2string().contains("UNKNOWN")).toList();
-
-        if (!missingTokens.isEmpty()) {
-            var typeList = missingTokens.stream().map(Token::getType).map(Object::toString).collect(Collectors.joining(", "));
-            fail("Found token types with no string representation: %s".formatted(typeList));
-        }
-
-    }
-
     /**
      * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
      * @param fileName a code sample file name
      * @param tokens the list of Tokens generated from the sample
      */
-    private void testSourceCoverage(String fileName, TokenList tokens) {
+    private void testSourceCoverage(String fileName, List<Token> tokens) {
         File testFile = new File(testFileLocation, fileName);
 
         try {
@@ -106,13 +89,13 @@ class SwiftFrontendTest {
             // All lines that contain code
             var codeLines = getCodeLines(lines);
             // All lines that contain token
-            var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().toArray();
+            var tokenLines = tokens.stream().mapToInt(Token::getLine).filter(line -> line != Token.NO_VALUE).distinct().toArray();
 
             if (codeLines.length > tokenLines.length) {
                 var diffLine = IntStream.range(0, codeLines.length)
                         .dropWhile(lineIdx -> lineIdx < tokenLines.length && codeLines[lineIdx] == tokenLines[lineIdx]).findFirst();
-                diffLine.ifPresent(
-                        lineIdx -> fail("Line %d of file '%s' is not represented in the token list.".formatted(codeLines[lineIdx], fileName)));
+                assertTrue(diffLine.isEmpty(),
+                        "Line %d of file '%s' is not represented in the token list.".formatted(codeLines[diffLine.getAsInt()], fileName));
             }
             assertArrayEquals(codeLines, tokenLines);
         } catch (IOException exception) {
@@ -158,18 +141,14 @@ class SwiftFrontendTest {
      * @param tokens TokenList which is supposed to contain all types of tokens
      * @param fileName The file name of the complete code example
      */
-    private void testTokenCoverage(TokenList tokens, String fileName) {
-        var foundTokens = StreamSupport.stream(tokens.allTokens().spliterator(), true).mapToInt(Token::getType).sorted().distinct().toArray();
-        // Exclude SEPARATOR_TOKEN, as it does not occur
-        var allTokens = IntStream.range(0, SwiftTokenConstants.NUMBER_DIFF_TOKENS).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).toArray();
-
-        if (allTokens.length > foundTokens.length) {
-            var diffLine = IntStream.range(0, allTokens.length)
-                    .dropWhile(lineIdx -> lineIdx < foundTokens.length && allTokens[lineIdx] == foundTokens[lineIdx]).findFirst();
-            diffLine.ifPresent(lineIdx -> fail("Token type %s was not found in the complete code example '%s'."
-                    .formatted(new SwiftToken(allTokens[lineIdx], fileName, -1, -1, -1).type2string(), fileName)));
-        }
-        assertArrayEquals(allTokens, foundTokens);
+    private void testTokenCoverage(List<Token> tokens, String fileName) {
+        var annotatedTokens = tokens.stream().map(Token::getType).collect(Collectors.toSet());
+        assertTrue(annotatedTokens.contains(SharedTokenType.FILE_END));
+        var annotatedSwiftTokens = annotatedTokens.stream().filter(SwiftTokenType.class::isInstance).collect(Collectors.toSet());
+        var allSwiftTokens = SwiftTokenType.values();
+        var missingSwiftTokens = Arrays.stream(allSwiftTokens).filter(token -> !annotatedSwiftTokens.contains(token)).toList();
+        assertTrue(missingSwiftTokens.isEmpty(), "The following Swift tokens are missing in the code example '%s':\n".formatted(fileName)
+                + String.join("\n", missingSwiftTokens.stream().map(SwiftTokenType::getDescription).toList()));
     }
 
 }