Enrich the C/C++ and Scheme tokens with column and length information, thus making the lexicographic token obsolete.

jplag.frontend-utils/src/main/java/de/jplag/Language.java

@@ -43,8 +43,7 @@ public interface Language {
     public int errorCount();
 
     /**
-     * Determines whether the parser provide column information. If this returns true, all generated tokens are a
-     * {@link TypographicToken}.
+     * Determines whether the parser provide column information.
      */
     public boolean supportsColumns();
 

jplag.frontend-utils/src/main/java/de/jplag/Token.java

@@ -7,11 +7,13 @@ package de.jplag;
  */
 public abstract class Token {
     private boolean basecode = false;
+    private int column;
     private String file;
     private int hash = -1;// hash-value. set and used by main algorithm (GSTiling)
-    private boolean marked;
 
+    private int length;
     private int line;
+    private boolean marked;
     protected int type;
 
     /**
@@ -26,6 +28,20 @@ public abstract class Token {
         setLine(line > 0 ? line : 1);
     }
 
+    /**
+     * Creates a token with column and length information.
+     * @param type is the token type.
+     * @param file is the name of the source code file.
+     * @param line is the line in the source code where the token resides.
+     * @param column is the line index in the line, meaning where the token starts
+     * @param length is the length of the token in the line.
+     */
+    public Token(int type, String file, int line, int column, int length) {
+        this(type, file, line);
+        this.column = column;
+        this.length = length;
+    }
+
     public String getFile() {
         return file;
     }
@@ -44,11 +60,11 @@ public abstract class Token {
     }
 
     public int getColumn() {
-        return 0; // no column
+        return column;
     }
 
     public int getLength() {
-        return 0; // no length
+        return length;
     }
 
     public boolean isBasecode() {
@@ -63,6 +79,14 @@ public abstract class Token {
         this.line = line;
     }
 
+    public void setColumn(int column) {
+        this.column = column;
+    }
+
+    public void setLength(int length) {
+        this.length = length;
+    }
+
     @Override
     public String toString() {
         return type2string();

jplag.frontend-utils/src/main/java/de/jplag/TypographicToken.java

@@ -1,44 +0,0 @@
-package de.jplag;
-
-/**
- * A token that tracks typographic information in the underlying source code, such as the column of the token in the
- * line and the length of the token in the line.
- * @author Timur Saglam
- */
-public abstract class TypographicToken extends Token {
-    private int column;
-    private int length;
-
-    /**
-     * Creates a typographic token.
-     * @param type is the token type.
-     * @param file is the name of the source code file.
-     * @param line is the line in the source code where the token resides.
-     * @param column is the line index in the line, meaning where the token starts
-     * @param length is the length of the token in the line.
-     */
-    public TypographicToken(int type, String file, int line, int column, int length) {
-        super(type, file, line);
-        this.column = column;
-        this.length = length;
-    }
-
-    @Override
-    public int getColumn() {
-        return column;
-    }
-
-    @Override
-    public int getLength() {
-        return length;
-    }
-
-    public void setColumn(int column) {
-        this.column = column;
-    }
-
-    public void setLength(int length) {
-        this.length = length;
-    }
-
-}

jplag.frontend.chars/src/main/java/de/jplag/chars/CharToken.java

@@ -32,6 +32,16 @@ public class CharToken extends Token {
         // Do nothing!
     }
 
+    @Override
+    public void setColumn(int column) {
+        // Do nothing!
+    }
+
+    @Override
+    public void setLength(int length) {
+        // Do nothing!
+    }
+
     @Override
     protected String type2string() {
         if (character == null) {

jplag.frontend.cpp/src/main/java/de/jplag/cpp/CPPToken.java

@@ -4,8 +4,12 @@ import de.jplag.Token;
 
 public class CPPToken extends Token implements CPPTokenConstants {
 
-    public CPPToken(int type, String file, int line) {
-        super(type, file, line);
+    public CPPToken(int type, String file, int line, int column, int length) {
+        super(type, file, line, column, length);
+    }
+
+    public CPPToken(int type, String file) {
+        super(type, file, 1, 0, 0);
     }
 
     protected String type2string() {
@@ -137,7 +141,7 @@ public class CPPToken extends Token implements CPPTokenConstants {
                 return "NULL";
 
             default:
-                return "<UNBEKANNT>";
+                return "<UNKNOWN>";
         }
     }
 }

jplag.frontend.cpp/src/main/java/de/jplag/cpp/Scanner.java

@@ -6,26 +6,28 @@ import de.jplag.AbstractParser;
 import de.jplag.TokenList;
 
 public class Scanner extends AbstractParser implements CPPTokenConstants {
-    private String actFile;
+    private String currentFile;
 
-    private TokenList struct;
+    private TokenList tokens;
 
-    public TokenList scan(File dir, String files[]) {
-        struct = new TokenList();
+    public TokenList scan(File directory, String files[]) {
+        tokens = new TokenList();
         errors = 0;
         CPPScanner scanner = null;// will be initialized in Method scanFile
         for (int i = 0; i < files.length; i++) {
-            actFile = files[i];
+            currentFile = files[i];
             getErrorConsumer().print(null, "Scanning file " + files[i]);
-            if (!CPPScanner.scanFile(dir, files[i], scanner, this))
+            if (!CPPScanner.scanFile(directory, files[i], scanner, this)) {
                 errors++;
-            struct.addToken(new CPPToken(FILE_END, actFile, 1));
+            }
+            tokens.addToken(new CPPToken(FILE_END, currentFile));
         }
         this.parseEnd();
-        return struct;
+        return tokens;
     }
 
     public void add(int type, Token token) {
-        struct.addToken(new CPPToken(type, actFile, token.beginLine));
+        int length = token.endColumn - token.beginColumn + 1;
+        tokens.addToken(new CPPToken(type, currentFile, token.beginLine, token.beginColumn, length));
     }
 }

jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/CSharpToken.java

@@ -1,8 +1,8 @@
 package de.jplag.csharp;
 
-import de.jplag.TypographicToken;
+import de.jplag.Token;
 
-public class CSharpToken extends TypographicToken implements CSharpTokenConstants {
+public class CSharpToken extends Token implements CSharpTokenConstants {
 
     public CSharpToken(int type, String file, int line, int column, int length) {
         super(type, file, line, column, length);

jplag.frontend.java/src/main/java/de/jplag/java/JavaToken.java

@@ -1,8 +1,8 @@
 package de.jplag.java;
 
-import de.jplag.TypographicToken;
+import de.jplag.Token;
 
-public class JavaToken extends TypographicToken implements JavaTokenConstants {
+public class JavaToken extends Token implements JavaTokenConstants {
 
     public JavaToken(int type, String file, int line, int col, int length) {
         super(type, file, line, col, length);
@@ -16,134 +16,134 @@ public class JavaToken extends TypographicToken implements JavaTokenConstants {
             case JavaTokenConstants.SEPARATOR_TOKEN:
                 return "METHOD_SEPARATOR";
             case J_PACKAGE:
-                return "PACKAGE ";
+                return "PACKAGE";
             case J_IMPORT:
-                return "IMPORT  ";
+                return "IMPORT";
             case J_CLASS_BEGIN:
-                return "CLASS{  ";
+                return "CLASS{";
             case J_CLASS_END:
-                return "}CLASS  ";
+                return "}CLASS";
             case J_METHOD_BEGIN:
-                return "METHOD{ ";
+                return "METHOD{";
             case J_METHOD_END:
-                return "}METHOD ";
+                return "}METHOD";
             case J_VARDEF:
-                return "VARDEF  ";
+                return "VARDEF";
             case J_SYNC_BEGIN:
-                return "SYNC{   ";
+                return "SYNC{";
             case J_SYNC_END:
-                return "}SYNC   ";
+                return "}SYNC";
             case J_DO_BEGIN:
-                return "DO{     ";
+                return "DO{";
             case J_DO_END:
-                return "}DO     ";
+                return "}DO";
             case J_WHILE_BEGIN:
-                return "WHILE{  ";
+                return "WHILE{";
             case J_WHILE_END:
-                return "}WHILE  ";
+                return "}WHILE";
             case J_FOR_BEGIN:
-                return "FOR{    ";
+                return "FOR{";
             case J_FOR_END:
-                return "}FOR    ";
+                return "}FOR";
             case J_SWITCH_BEGIN:
-                return "SWITCH{ ";
+                return "SWITCH{";
             case J_SWITCH_END:
-                return "}SWITCH ";
+                return "}SWITCH";
             case J_CASE:
-                return "CASE    ";
+                return "CASE";
             case J_TRY_BEGIN:
-                return "TRY{    ";
+                return "TRY{";
             case J_CATCH_BEGIN:
-                return "CATCH{  ";
+                return "CATCH{";
             case J_CATCH_END:
-                return "}CATCH  ";
+                return "}CATCH";
             case J_FINALLY:
-                return "FINALLY ";
+                return "FINALLY";
             case J_IF_BEGIN:
-                return "IF{     ";
+                return "IF{";
             case J_ELSE:
-                return "ELSE    ";
+                return "ELSE";
             case J_IF_END:
-                return "}IF     ";
+                return "}IF";
             case J_COND:
-                return "COND    ";
+                return "COND";
             case J_BREAK:
-                return "BREAK   ";
+                return "BREAK";
             case J_CONTINUE:
                 return "CONTINUE";
             case J_RETURN:
-                return "RETURN  ";
+                return "RETURN";
             case J_THROW:
-                return "THROW   ";
+                return "THROW";
             case J_IN_CLASS_BEGIN:
                 return "INCLASS{";
             case J_IN_CLASS_END:
                 return "}INCLASS";
             case J_APPLY:
-                return "APPLY   ";
+                return "APPLY";
             case J_NEWCLASS:
                 return "NEWCLASS";
             case J_NEWARRAY:
                 return "NEWARRAY";
             case J_ASSIGN:
-                return "ASSIGN  ";
+                return "ASSIGN";
             case J_INTERFACE_BEGIN:
-                return "INTERF{ ";
+                return "INTERF{";
             case J_INTERFACE_END:
-                return "}INTERF ";
+                return "}INTERF";
             case J_CONSTR_BEGIN:
-                return "CONSTR{ ";
+                return "CONSTR{";
             case J_CONSTR_END:
-                return "}CONSTR ";
+                return "}CONSTR";
             case J_INIT_BEGIN:
-                return "INIT{   ";
+                return "INIT{";
             case J_INIT_END:
-                return "}INIT   ";
+                return "}INIT";
             case J_VOID:
-                return "VOID    ";
+                return "VOID";
             case J_ARRAY_INIT_BEGIN:
                 return "ARRINIT{";
             case J_ARRAY_INIT_END:
                 return "ARRINIT}";
             case J_ENUM_BEGIN:
-                return "ENUM{   ";
+                return "ENUM{";
             case J_ENUM_CLASS_BEGIN:
                 return "ENUM_CLA";
             case J_ENUM_END:
-                return "}ENUM   ";
+                return "}ENUM";
             case J_GENERIC:
-                return "GENERIC ";
+                return "GENERIC";
             case J_ASSERT:
-                return "ASSERT  ";
+                return "ASSERT";
             case J_ANNO:
-                return "ANNO    ";
+                return "ANNO";
             case J_ANNO_MARKER:
                 return "ANNOMARK";
             case J_ANNO_M_BEGIN:
-                return "ANNO_M{ ";
+                return "ANNO_M{";
             case J_ANNO_M_END:
-                return "}ANNO_M ";
+                return "}ANNO_M";
             case J_ANNO_T_BEGIN:
-                return "ANNO_T{ ";
+                return "ANNO_T{";
             case J_ANNO_T_END:
-                return "}ANNO_T ";
+                return "}ANNO_T";
             case J_ANNO_C_BEGIN:
-                return "ANNO_C{ ";
+                return "ANNO_C{";
             case J_ANNO_C_END:
-                return "}ANNO_C ";
+                return "}ANNO_C";
             case J_MODULE_BEGIN:
-                return "MODULE{ ";
+                return "MODULE{";
             case J_MODULE_END:
-                return "}MODULE ";
+                return "}MODULE";
             case J_EXPORTS:
-                return "EXPORTS ";
+                return "EXPORTS";
             case J_PROVIDES:
                 return "PROVIDES";
             case J_REQUIRES:
                 return "REQUIRES";
 
             case J_TRY_WITH_RESOURCE:
-                return "TRY_RES ";
+                return "TRY_RES";
 
             default:
                 System.err.println("*UNKNOWN: " + type);

jplag.frontend.python-3/src/main/java/de/jplag/python3/Python3Token.java

@@ -1,8 +1,8 @@
 package de.jplag.python3;
 
-import de.jplag.TypographicToken;
+import de.jplag.Token;
 
-public class Python3Token extends TypographicToken implements Python3TokenConstants {
+public class Python3Token extends Token implements Python3TokenConstants {
 
     public Python3Token(int type, String file, int line, int column, int length) {
         super(type, file, line, column, length);

jplag.frontend.scheme/src/main/java/de/jplag/scheme/Parser.java

@@ -6,31 +6,27 @@ import de.jplag.AbstractParser;
 import de.jplag.TokenList;
 
 public class Parser extends AbstractParser implements SchemeTokenConstants {
-    private String actFile;
+    private String currentFile;
 
     private TokenList struct;
 
-    public TokenList parse(File dir, String files[]) {
+    public TokenList parse(File directory, String files[]) {
         struct = new TokenList();
         errors = 0;
-        SchemeParser parser = null;// no worry it will be reinitialized
-        // in method parseFile(...)
         for (int i = 0; i < files.length; i++) {
-            actFile = files[i];
+            currentFile = files[i];
             getErrorConsumer().print(null, "Parsing file " + files[i]);
-            if (!SchemeParser.parseFile(dir, files[i], parser, this))
+            if (!SchemeParser.parseFile(directory, files[i], null, this))
                 errors++;
-            struct.addToken(new SchemeToken(FILE_END, actFile, 1));
+            struct.addToken(new SchemeToken(FILE_END, currentFile));
         }
         this.parseEnd();
         return struct;
     }
 
     public void add(int type, Token token) {
-        struct.addToken(new SchemeToken(type, actFile, token.beginLine));
-        /*
-         * System.out.println(token.beginLine+"\t"+ (new SchemeToken(0,null,0)).type2string(type)+"\t"+ token.image);
-         */
+        int length = token.endColumn - token.beginColumn + 1;
+        struct.addToken(new SchemeToken(type, currentFile, token.beginLine, token.endLine, length));
     }
 
 }

jplag.frontend.scheme/src/main/java/de/jplag/scheme/SchemeToken.java

@@ -4,8 +4,12 @@ import de.jplag.Token;
 
 public class SchemeToken extends Token implements SchemeTokenConstants {
 
-    public SchemeToken(int type, String file, int line) {
-        super(type, file, line);
+    public SchemeToken(int type, String file, int line, int column, int length) {
+        super(type, file, line, column, length);
+    }
+
+    public SchemeToken(int type, String file) {
+        super(type, file, 1, 0, 0);
     }
 
     protected String type2string() {

jplag.frontend.text/src/main/java/de/jplag/text/TextToken.java

@@ -1,8 +1,8 @@
 package de.jplag.text;
 
-import de.jplag.TypographicToken;
+import de.jplag.Token;
 
-public class TextToken extends TypographicToken {
+public class TextToken extends Token {
     private static final long serialVersionUID = 4301179216570538972L;
 
     private static int getSerial(String text, Parser parser) {