Distinguished two types of tokens. Typographic tokens that provide column and length with the line index, and normal ones which do not.

jplag.frontend-utils/src/main/java/de/jplag/Token.java

@@ -1,30 +1,29 @@
 package de.jplag;
 
+/**
+ * This class represents a token in a source code. It can represents keywords, identifies, syntactical structures etc.
+ * What types of tokens there are depends on the specific language, meaning JPlag does not enforce a specific token set.
+ * The language parsers decide what is a token and what is not.
+ */
 public abstract class Token {
     private boolean basecode = false;
     private String file;
     private int hash = -1;// hash-value. set and used by main algorithm (GSTiling)
     private boolean marked;
 
-    protected int column;
-    protected int length;
-    protected int line;
+    private int line;
     protected int type;
 
+    /**
+     * Creates a token without information about the column or the length of the token in the line.
+     * @param type is the token type.
+     * @param file is the name of the source code file.
+     * @param line is the line in the source code where the token resides. Cannot be smaller than 1.
+     */
     public Token(int type, String file, int line) {
-        this(type, file, line, -1, -1);
-    }
-
-    public Token(int type, String file, int line, int column, int length) {
         this.type = type;
         this.file = file;
         setLine(line > 0 ? line : 1);
-        setColumn(column);
-        setLength(length);
-    }
-
-    public int getColumn() {
-        return column;
     }
 
     public String getFile() {
@@ -40,28 +39,24 @@ public abstract class Token {
         return -1;
     }
 
-    public int getLength() {
-        return length;
-    }
-
     public int getLine() {
         return line;
     }
 
-    public boolean isBasecode() {
-        return basecode;
+    public int getColumn() {
+        return 0; // no column
     }
 
-    public boolean isMarked() {
-        return marked;
+    public int getLength() {
+        return 0; // no length
     }
 
-    public void setColumn(int column) {
-        this.column = column;
+    public boolean isBasecode() {
+        return basecode;
     }
 
-    public void setLength(int length) {
-        this.length = length;
+    public boolean isMarked() {
+        return marked;
     }
 
     public void setLine(int line) {

jplag.frontend-utils/src/main/java/de/jplag/TypographicToken.java

@@ -0,0 +1,44 @@
+package de.jplag;
+
+/**
+ * A token that tracks typographic information in the underlying source code, such as the column of the token in the
+ * line and the length of the token in the line.
+ * @author Timur Saglam
+ */
+public abstract class TypographicToken extends Token {
+    private int column;
+    private int length;
+
+    /**
+     * Creates a typographic token.
+     * @param type is the token type.
+     * @param file is the name of the source code file.
+     * @param line is the line in the source code where the token resides.
+     * @param column is the line index in the line, meaning where the token starts
+     * @param length is the length of the token in the line.
+     */
+    public TypographicToken(int type, String file, int line, int column, int length) {
+        super(type, file, line);
+        this.column = column;
+        this.length = length;
+    }
+
+    @Override
+    public int getColumn() {
+        return column;
+    }
+
+    @Override
+    public int getLength() {
+        return length;
+    }
+
+    public void setColumn(int column) {
+        this.column = column;
+    }
+
+    public void setLength(int length) {
+        this.length = length;
+    }
+
+}

jplag.frontend.chars/src/main/java/de/jplag/chars/CharToken.java

@@ -22,16 +22,6 @@ public class CharToken extends Token {
         return index; // so that the result-index is sorted correctly
     }
 
-    @Override
-    public int getColumn() {
-        return 0;
-    }
-
-    @Override
-    public int getLength() {
-        return 0;
-    }
-
     @Override
     public int getIndex() {
         return index;
@@ -42,16 +32,6 @@ public class CharToken extends Token {
         // Do nothing!
     }
 
-    @Override
-    public void setColumn(int column) {
-        // Do nothing!
-    }
-
-    @Override
-    public void setLength(int length) {
-        // Do nothing!
-    }
-
     @Override
     protected String type2string() {
         if (character == null) {

jplag.frontend.cpp/src/main/java/de/jplag/cpp/CPPToken.java

@@ -8,26 +8,6 @@ public class CPPToken extends Token implements CPPTokenConstants {
         super(type, file, line);
     }
 
-    @Override
-    public int getColumn() {
-        return 0;
-    }
-
-    @Override
-    public int getLength() {
-        return 0;
-    }
-
-    @Override
-    public void setColumn(int column) {
-        // Do nothing!
-    }
-
-    @Override
-    public void setLength(int length) {
-        // Do nothing!
-    }
-
     protected String type2string() {
         switch (type) {
             case CPPTokenConstants.FILE_END:

jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/CSharpToken.java

@@ -1,8 +1,8 @@
 package de.jplag.csharp;
 
-import de.jplag.Token;
+import de.jplag.TypographicToken;
 
-public class CSharpToken extends Token implements CSharpTokenConstants {
+public class CSharpToken extends TypographicToken implements CSharpTokenConstants {
 
     public CSharpToken(int type, String file, int line, int column, int length) {
         super(type, file, line, column, length);

jplag.frontend.java/src/main/java/de/jplag/java/JavaToken.java

@@ -1,8 +1,8 @@
 package de.jplag.java;
 
-import de.jplag.Token;
+import de.jplag.TypographicToken;
 
-public class JavaToken extends Token implements JavaTokenConstants {
+public class JavaToken extends TypographicToken implements JavaTokenConstants {
 
     public JavaToken(int type, String file, int line, int col, int length) {
         super(type, file, line, col, length);

jplag.frontend.python-3/src/main/java/de/jplag/python3/Python3Token.java

@@ -1,8 +1,8 @@
 package de.jplag.python3;
 
-import de.jplag.Token;
+import de.jplag.TypographicToken;
 
-public class Python3Token extends Token implements Python3TokenConstants {
+public class Python3Token extends TypographicToken implements Python3TokenConstants {
 
     public Python3Token(int type, String file, int line, int column, int length) {
         super(type, file, line, column, length);

jplag.frontend.scheme/src/main/java/de/jplag/scheme/SchemeToken.java

@@ -8,26 +8,6 @@ public class SchemeToken extends Token implements SchemeTokenConstants {
         super(type, file, line);
     }
 
-    @Override
-    public int getColumn() {
-        return 0;
-    }
-
-    @Override
-    public int getLength() {
-        return 0;
-    }
-
-    @Override
-    public void setColumn(int line) {
-        // Do nothing!
-    }
-
-    @Override
-    public void setLength(int line) {
-        // Do nothing!
-    }
-
     protected String type2string() {
         switch (type) {
             case SchemeTokenConstants.FILE_END:

jplag.frontend.text/src/main/java/de/jplag/text/TextToken.java

@@ -1,8 +1,8 @@
 package de.jplag.text;
 
-import de.jplag.Token;
+import de.jplag.TypographicToken;
 
-public class TextToken extends Token {
+public class TextToken extends TypographicToken {
     private static final long serialVersionUID = 4301179216570538972L;
 
     private static int getSerial(String text, Parser parser) {