Clean up tokens and token constants, reduce code duplication, remove unused code.

jplag.frontend-utils/src/main/java/de/jplag/Language.java

@@ -61,9 +61,4 @@ public interface Language {
      * Number of defined tokens in the scanner of the language.
      */
     public int numberOfTokens();
-
-    /**
-     * Convert a token type to a text representation.
-     */
-    public String type2string(int type);
 }

jplag.frontend-utils/src/main/java/de/jplag/Token.java

@@ -1,13 +1,12 @@
 package de.jplag;
 
-import java.io.Serializable;
-
-abstract public class Token implements TokenConstants, Serializable {
-    private static final long serialVersionUID = 862935679966383302L;
+abstract public class Token {
     public int type;
     public String file;
 
-    protected boolean marked;
+    protected int line, column, length;
+
+    protected boolean marked; // TODO TS: Why protected/no accessors?
     protected boolean basecode = false;
     protected int hash = -1;// hash-value. set and used by main algorithm (GSTiling)
 
@@ -23,20 +22,28 @@ abstract public class Token implements TokenConstants, Serializable {
         setLength(length);
     }
 
-    // abstract members
-    abstract public int getLine();
+    public int getLine() {
+        return line;
+    }
 
-    abstract public int getColumn();
+    public int getColumn() {
+        return column;
+    }
 
-    abstract public int getLength();
+    public int getLength() {
+        return length;
+    }
 
-    protected void setLine(int line) {
+    public void setLine(int line) {
+        this.line = line;
     }
 
-    protected void setColumn(int line) {
+    public void setColumn(int column) {
+        this.column = column;
     }
 
-    protected void setLength(int line) {
+    public void setLength(int length) {
+        this.length = length;
     }
 
     // this is made to distinguish the character front end. Maybe other front ends can use it too?
@@ -44,12 +51,10 @@ abstract public class Token implements TokenConstants, Serializable {
         return -1;
     }
 
-    public static String type2string(int type) {
-        return "<abstract>";
-    }
+    protected abstract String type2string();
 
     @Override
     public String toString() {
-        return type2string(type);
+        return type2string();
     }
 }

jplag.frontend-utils/src/main/java/de/jplag/TokenConstants.java

@@ -2,6 +2,7 @@ package de.jplag;
 
 public interface TokenConstants {
     final static int FILE_END = 0;
+
     /**
      * Used to optionally separate methods from each other with an always marked token
      */

jplag.frontend.chars/src/main/java/de/jplag/chars/CharToken.java

@@ -3,17 +3,18 @@ package de.jplag.chars;
 import de.jplag.Token;
 
 public class CharToken extends Token {
-    private static final long serialVersionUID = 1L;
 
     private int index;
+    private Character character;
 
-    public CharToken(int type, String file, int index, Parser parser) {
+    public CharToken(int type, String file, int index, Character character, Parser parser) {
         super(type, file, -1);
         this.index = index;
+        this.character = character;
     }
 
     public CharToken(int type, String file, Parser parser) {
-        this(type, file, -1, parser);
+        this(type, file, -1, null, parser);
     }
 
     @Override
@@ -35,4 +36,27 @@ public class CharToken extends Token {
     public int getIndex() {
         return index;
     }
+
+    @Override
+    public void setLine(int line) {
+        // Do nothing!
+    }
+
+    @Override
+    public void setColumn(int column) {
+        // Do nothing!
+    }
+
+    @Override
+    public void setLength(int length) {
+        // Do nothing!
+    }
+
+    @Override
+    protected String type2string() {
+        if (character == null) {
+            return "FILE_END";
+        }
+        return character.toString();
+    }
 }

jplag.frontend.chars/src/main/java/de/jplag/chars/Language.java

@@ -3,7 +3,6 @@ package de.jplag.chars;
 import java.io.File;
 
 import de.jplag.ErrorConsumer;
-import de.jplag.Token;
 import de.jplag.TokenList;
 
 /*
@@ -74,9 +73,4 @@ public class Language implements de.jplag.Language {
     public int numberOfTokens() {
         return 36;
     }
-
-    @Override
-    public String type2string(int type) {
-        return Token.type2string(type);
-    }
 }

jplag.frontend.chars/src/main/java/de/jplag/chars/Parser.java

@@ -44,7 +44,7 @@ public class Parser extends AbstractParser implements TokenConstants {
 
                 for (int i = 0; i < length; i++) {
                     if (buffer[i] <= 127 && (type = mapping[buffer[i]]) > 1) {
-                        struct.addToken(new CharToken(type, file, offset + i, this));
+                        struct.addToken(new CharToken(type, file, offset + i, buffer[i], this));
                     }
                 }
                 offset += length;

jplag.frontend.cpp/src/main/java/de/jplag/cpp/CPPToken.java

@@ -3,34 +3,32 @@ package de.jplag.cpp;
 import de.jplag.Token;
 
 public class CPPToken extends Token implements CPPTokenConstants {
-    private static final long serialVersionUID = 1L;
-    private int line;
 
     public CPPToken(int type, String file, int line) {
         super(type, file, line);
     }
 
     @Override
-    public int getLine() {
-        return line;
+    public int getColumn() {
+        return 0;
     }
 
     @Override
-    public void setLine(int line) {
-        this.line = line;
+    public int getLength() {
+        return 0;
     }
 
     @Override
-    public int getColumn() {
-        return 0;
+    public void setColumn(int column) {
+        // Do nothing!
     }
 
     @Override
-    public int getLength() {
-        return 0;
+    public void setLength(int length) {
+        // Do nothing!
     }
 
-    public static String type2string(int type) {
+    protected String type2string() {
         switch (type) {
             case CPPTokenConstants.FILE_END:
                 return "********";

jplag.frontend.cpp/src/main/java/de/jplag/cpp/CPPTokenConstants.java

@@ -3,11 +3,6 @@ package de.jplag.cpp;
 import de.jplag.TokenConstants;
 
 public interface CPPTokenConstants extends TokenConstants {
-    final static int FILE_END = 0;
-    // Used to optionally separate methods from each other
-    // with an always marked token
-    final static int SEPARATOR_TOKEN = 1;
-
     final static int C_BLOCK_BEGIN = 2;
     final static int C_BLOCK_END = 3;
     final static int C_SCOPE = 4;

jplag.frontend.cpp/src/main/java/de/jplag/cpp/Language.java

@@ -72,9 +72,4 @@ public class Language implements de.jplag.Language {
     public int numberOfTokens() {
         return CPPTokenConstants.NUM_DIFF_TOKENS;
     }
-
-    @Override
-    public String type2string(int type) {
-        return CPPToken.type2string(type);
-    }
 }

jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/CSharpToken.java

@@ -3,44 +3,12 @@ package de.jplag.csharp;
 import de.jplag.Token;
 
 public class CSharpToken extends Token implements CSharpTokenConstants {
-    private static final long serialVersionUID = 1L;
-    private int line, column, length;
 
     public CSharpToken(int type, String file, int line, int column, int length) {
         super(type, file, line, column, length);
     }
 
-    @Override
-    public int getLine() {
-        return line;
-    }
-
-    @Override
-    public int getColumn() {
-        return column;
-    }
-
-    @Override
-    public int getLength() {
-        return length;
-    }
-
-    @Override
-    public void setLine(int line) {
-        this.line = line;
-    }
-
-    @Override
-    public void setColumn(int column) {
-        this.column = column;
-    }
-
-    @Override
-    public void setLength(int length) {
-        this.length = length;
-    }
-
-    public static String type2string(int type) {
+    protected String type2string() {
         switch (type) {
             case CSharpTokenConstants.FILE_END:
                 return "**********";

jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/CSharpTokenConstants.java

@@ -3,12 +3,6 @@ package de.jplag.csharp;
 import de.jplag.TokenConstants;
 
 public interface CSharpTokenConstants extends TokenConstants {
-    final static int FILE_END = 0;
-
-    // Used to optionally separate methods from each other
-    // with an always marked token
-    final static int SEPARATOR_TOKEN = 1;
-
     final static int _INVOCATION = 2;
     final static int _OBJECT_CREATION = 3;
     final static int _ARRAY_CREATION = 4;

jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/Language.java

@@ -69,9 +69,4 @@ public class Language implements de.jplag.Language {
     public int numberOfTokens() {
         return CSharpTokenConstants.NUM_DIFF_TOKENS;
     }
-
-    @Override
-    public String type2string(int type) {
-        return CSharpToken.type2string(type);
-    }
 }

jplag.frontend.java/src/main/java/de/jplag/java/JavaToken.java

@@ -3,44 +3,13 @@ package de.jplag.java;
 import de.jplag.Token;
 
 public class JavaToken extends Token implements JavaTokenConstants {
-    private static final long serialVersionUID = -383581430479870696L;
-    private int line, column, length;
 
     public JavaToken(int type, String file, int line, int col, int length) {
         super(type, file, line, col, length);
     }
 
     @Override
-    public int getLine() {
-        return line;
-    }
-
-    @Override
-    public int getColumn() {
-        return column;
-    }
-
-    @Override
-    public int getLength() {
-        return length;
-    }
-
-    @Override
-    public void setLine(int line) {
-        this.line = line;
-    }
-
-    @Override
-    public void setColumn(int column) {
-        this.column = column;
-    }
-
-    @Override
-    public void setLength(int length) {
-        this.length = length;
-    }
-
-    public static String type2string(int type) {
+    protected String type2string() {
         switch (type) {
             case JavaTokenConstants.FILE_END:
                 return "********";

jplag.frontend.java/src/main/java/de/jplag/java/JavaTokenConstants.java

@@ -3,11 +3,8 @@ package de.jplag.java;
 import de.jplag.TokenConstants;
 
 public interface JavaTokenConstants extends TokenConstants {
-    final static int FILE_END = 0;
-
-    // check used to mark tokens that are actually generated by the adapter
-    final static int SEPARATOR_TOKEN = 1;// @formatter:off
 
+    // @formatter:off
     final static int J_PACKAGE = 2;				// check
     final static int J_IMPORT = 3;				// check
     final static int J_CLASS_BEGIN = 4;			// check

jplag.frontend.java/src/main/java/de/jplag/java/Language.java

@@ -57,11 +57,6 @@ public class Language implements de.jplag.Language {
         return JavaTokenConstants.NUM_DIFF_TOKENS;
     }
 
-    @Override
-    public String type2string(int type) {
-        return JavaToken.type2string(type);
-    }
-
     @Override
     public TokenList parse(File dir, String[] files) {
         return this.parser.parse(dir, files);

jplag.frontend.python-3/src/main/java/de/jplag/python3/Language.java

@@ -69,9 +69,4 @@ public class Language implements de.jplag.Language {
     public int numberOfTokens() {
         return Python3TokenConstants.NUM_DIFF_TOKENS;
     }
-
-    @Override
-    public String type2string(int type) {
-        return Python3Token.type2string(type);
-    }
 }

jplag.frontend.python-3/src/main/java/de/jplag/python3/Python3Token.java

@@ -4,45 +4,11 @@ import de.jplag.Token;
 
 public class Python3Token extends Token implements Python3TokenConstants {
 
-    private static final long serialVersionUID = 1485877548175917943L;
-
-    private int line, column, length;
-
     public Python3Token(int type, String file, int line, int column, int length) {
         super(type, file, line, column, length);
     }
 
-    @Override
-    public int getLine() {
-        return line;
-    }
-
-    @Override
-    public int getColumn() {
-        return column;
-    }
-
-    @Override
-    public int getLength() {
-        return length;
-    }
-
-    @Override
-    public void setLine(int line) {
-        this.line = line;
-    }
-
-    @Override
-    public void setColumn(int column) {
-        this.column = column;
-    }
-
-    @Override
-    public void setLength(int length) {
-        this.length = length;
-    }
-
-    public static String type2string(int type) {
+    protected String type2string() {
         switch (type) {
             case Python3TokenConstants.FILE_END:
                 return "********";

jplag.frontend.python-3/src/main/java/de/jplag/python3/Python3TokenConstants.java

@@ -3,10 +3,6 @@ package de.jplag.python3;
 import de.jplag.TokenConstants;
 
 public interface Python3TokenConstants extends TokenConstants {
-
-    final static int FILE_END = 0;
-    final static int SEPARATOR_TOKEN = 1;
-
     final static int IMPORT = 2;
     final static int CLASS_BEGIN = 3;
     final static int CLASS_END = 4;

jplag.frontend.scheme/src/main/java/de/jplag/scheme/Language.java

@@ -70,9 +70,4 @@ public class Language implements de.jplag.Language {
     public int numberOfTokens() {
         return SchemeTokenConstants.NUM_DIFF_TOKENS;
     }
-
-    @Override
-    public String type2string(int type) {
-        return SchemeToken.type2string(type);
-    }
 }

jplag.frontend.scheme/src/main/java/de/jplag/scheme/SchemeToken.java

@@ -3,34 +3,32 @@ package de.jplag.scheme;
 import de.jplag.Token;
 
 public class SchemeToken extends Token implements SchemeTokenConstants {
-    private static final long serialVersionUID = -143418583849687339L;
-    private int line;
 
     public SchemeToken(int type, String file, int line) {
         super(type, file, line);
     }
 
     @Override
-    public int getLine() {
-        return line;
+    public int getColumn() {
+        return 0;
     }
 
     @Override
-    public void setLine(int line) {
-        this.line = line;
+    public int getLength() {
+        return 0;
     }
 
     @Override
-    public int getColumn() {
-        return 0;
+    public void setColumn(int line) {
+        // Do nothing!
     }
 
     @Override
-    public int getLength() {
-        return 0;
+    public void setLength(int line) {
+        // Do nothing!
     }
 
-    public static String type2string(int type) {
+    protected String type2string() {
         switch (type) {
             case SchemeTokenConstants.FILE_END:
                 return "********";

jplag.frontend.scheme/src/main/java/de/jplag/scheme/SchemeTokenConstants.java

@@ -3,11 +3,6 @@ package de.jplag.scheme;
 import de.jplag.TokenConstants;
 
 public interface SchemeTokenConstants extends TokenConstants {
-    final static int FILE_END = 0;
-
-    // Used to optionally separate methods from each other
-    // with an always marked token
-    final static int SEPARATOR_TOKEN = 1;
 
     final static int S_BOOL = 2;
     final static int S_NUMBER = 3;

jplag.frontend.text/src/main/java/de/jplag/text/Language.java

@@ -4,7 +4,6 @@ package de.jplag.text;
 import java.io.File;
 
 import de.jplag.ErrorConsumer;
-import de.jplag.Token;
 import de.jplag.TokenList;
 
 public class Language implements de.jplag.Language {
@@ -73,9 +72,4 @@ public class Language implements de.jplag.Language {
     public int numberOfTokens() {
         return parser.serial;
     }
-
-    @Override
-    public String type2string(int type) {
-        return Token.type2string(type);
-    }
 }

jplag.frontend.text/src/main/java/de/jplag/text/TextToken.java

@@ -19,9 +19,6 @@ public class TextToken extends Token {
         return serial.intValue();
     }
 
-    // ///////////////////// END OF STATIC MEMBERS
-
-    private int line, column, length;
     private String text;
 
     public TextToken(int type, String file, Parser parser) {
@@ -34,37 +31,12 @@ public class TextToken extends Token {
         this.text = text.toLowerCase();
     }
 
-    @Override
-    public int getLine() {
-        return line;
-    }
-
-    @Override
-    public int getColumn() {
-        return column;
-    }
-
-    @Override
-    public int getLength() {
-        return length;
-    }
-
-    @Override
-    public void setLine(int line) {
-        this.line = line;
-    }
-
-    @Override
-    public void setColumn(int column) {
-        this.column = column;
+    public String getText() {
+        return this.text;
     }
 
     @Override
-    public void setLength(int length) {
-        this.length = length;
-    }
-
-    public String getText() {
-        return this.text;
+    protected String type2string() {
+        return getText();
     }
 }