Merge pull request #361 from jplag/tree-scanner

Overhaul the Java frontend

jplag.frontend-utils/src/main/java/de/jplag/AbstractParser.java

@@ -24,7 +24,7 @@ public abstract class AbstractParser {
         return errorConsumer;
     }
 
-    public void setProgram(ErrorConsumer errorConsumer) {
+    public void setErrorConsumer(ErrorConsumer errorConsumer) {
         this.errorConsumer = errorConsumer;
     }
 }

jplag.frontend.chars/src/main/java/de/jplag/chars/Language.java

@@ -13,7 +13,7 @@ public class Language implements de.jplag.Language {
     private final de.jplag.chars.Parser parser = new Parser();
 
     public Language(ErrorConsumer program) {
-        this.parser.setProgram(program);
+        this.parser.setErrorConsumer(program);
     }
 
     @Override

jplag.frontend.cpp/src/main/java/de/jplag/cpp/Language.java

@@ -10,7 +10,7 @@ public class Language implements de.jplag.Language {
 
     public Language(ErrorConsumer program) {
         this.scanner = new Scanner();
-        this.scanner.setProgram(program);
+        this.scanner.setErrorConsumer(program);
     }
 
     @Override

jplag.frontend.csharp-1.2/src/main/java/de/jplag/csharp/Language.java

@@ -10,7 +10,7 @@ public class Language implements de.jplag.Language {
 
     public Language(ErrorConsumer program) {
         this.parser = new Parser();
-        this.parser.setProgram(program);
+        this.parser.setErrorConsumer(program);
 
     }
 

jplag.frontend.java/src/main/java/de/jplag/java/JavaToken.java

@@ -1,6 +1,7 @@
 package de.jplag.java;
 
 import de.jplag.Token;
+import de.jplag.TokenConstants;
 
 public class JavaToken extends Token implements JavaTokenConstants {
 
@@ -11,9 +12,9 @@ public class JavaToken extends Token implements JavaTokenConstants {
     @Override
     protected String type2string() {
         switch (type) {
-            case JavaTokenConstants.FILE_END:
+            case TokenConstants.FILE_END:
                 return "EOF";
-            case JavaTokenConstants.SEPARATOR_TOKEN:
+            case TokenConstants.SEPARATOR_TOKEN:
                 return "METHOD_SEPARATOR";
             case J_PACKAGE:
                 return "PACKAGE";

jplag.frontend.java/src/main/java/de/jplag/java/JavacAdapter.java

@@ -9,7 +9,7 @@ import java.util.Collections;
 import javax.tools.Diagnostic;
 import javax.tools.DiagnosticCollector;
 import javax.tools.JavaCompiler;
-import javax.tools.JavaFileObject;
+import javax.tools.JavaCompiler.CompilationTask;
 import javax.tools.StandardJavaFileManager;
 import javax.tools.ToolProvider;
 
@@ -19,38 +19,52 @@ import com.sun.source.util.JavacTask;
 import com.sun.source.util.SourcePositions;
 import com.sun.source.util.Trees;
 
+import de.jplag.ErrorConsumer;
+import de.jplag.TokenConstants;
+
 public class JavacAdapter {
 
     private static final JavaCompiler javac = ToolProvider.getSystemJavaCompiler();
 
-    public int parseFiles(File directory, File[] pathedFiles, final Parser parser) {
+    public int parseFiles(File directory, Iterable<File> pathedFiles, final Parser parser) {
         final StandardJavaFileManager fileManager = javac.getStandardFileManager(null, null, StandardCharsets.UTF_8);
-        DiagnosticCollector<? super JavaFileObject> diagListen = new DiagnosticCollector<>();
-        final JavaCompiler.CompilationTask task = javac.getTask(null, fileManager, diagListen, null, null,
-                fileManager.getJavaFileObjects(pathedFiles));
-        Iterable<? extends CompilationUnitTree> asts = Collections.emptyList();
-        try {
-            asts = ((JavacTask) task).parse();
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
+        var listener = new DiagnosticCollector<>();
+        var javaFiles = fileManager.getJavaFileObjectsFromFiles(pathedFiles);
+        final CompilationTask task = javac.getTask(null, fileManager, listener, null, null, javaFiles);
         final Trees trees = Trees.instance(task);
         final SourcePositions positions = trees.getSourcePositions();
-        for (final CompilationUnitTree ast : asts) {
-            final String filename;
-            if (directory == null)
-                filename = ast.getSourceFile().getName();
-            else {
-                filename = Paths.get(directory.toURI()).relativize(Paths.get(ast.getSourceFile().toUri())).toString();
-            }
+        for (final CompilationUnitTree ast : executeCompilationTask(task)) {
+            final String filename = fileNameOf(directory, ast);
             final LineMap map = ast.getLineMap();
             ast.accept(new TokenGeneratingTreeScanner(filename, parser, map, positions, ast), null);
-            parser.add(JavaTokenConstants.FILE_END, filename, 1, -1, -1);
+            parser.add(TokenConstants.FILE_END, filename, 1, -1, -1);
         }
+        return processErrors(parser.getErrorConsumer(), listener);
+    }
+
+    private Iterable<? extends CompilationUnitTree> executeCompilationTask(final CompilationTask task) {
+        Iterable<? extends CompilationUnitTree> abstractSyntaxTrees = Collections.emptyList();
+        try {
+            abstractSyntaxTrees = ((JavacTask) task).parse();
+        } catch (IOException exception) {
+            exception.printStackTrace();
+        }
+        return abstractSyntaxTrees;
+    }
+
+    private String fileNameOf(File directory, final CompilationUnitTree ast) {
+        if (directory == null)
+            return ast.getSourceFile().getName();
+        else {
+            return Paths.get(directory.toURI()).relativize(Paths.get(ast.getSourceFile().toUri())).toString();
+        }
+    }
+
+    private int processErrors(final ErrorConsumer consumer, DiagnosticCollector<Object> listener) {
         int errors = 0;
-        for (Diagnostic<?> diagItem : diagListen.getDiagnostics()) {
-            if (diagItem.getKind() == javax.tools.Diagnostic.Kind.ERROR) {
-                parser.getErrorConsumer().addError(diagItem.toString());
+        for (Diagnostic<?> diagnosticItem : listener.getDiagnostics()) {
+            if (diagnosticItem.getKind() == javax.tools.Diagnostic.Kind.ERROR) {
+                consumer.addError(diagnosticItem.toString());
                 errors++;
             }
         }

jplag.frontend.java/src/main/java/de/jplag/java/Language.java

@@ -11,9 +11,9 @@ import de.jplag.TokenList;
 public class Language implements de.jplag.Language {
     private final Parser parser;
 
-    public Language(ErrorConsumer program) {
+    public Language(ErrorConsumer errorConsumer) {
         this.parser = new Parser();
-        this.parser.setProgram(program);
+        this.parser.setErrorConsumer(errorConsumer);
     }
 
     @Override
@@ -23,7 +23,7 @@ public class Language implements de.jplag.Language {
 
     @Override
     public String getName() {
-        return "Javac 1.9+ based AST plugin";
+        return "Javac based AST plugin";
     }
 
     @Override
@@ -57,8 +57,8 @@ public class Language implements de.jplag.Language {
     }
 
     @Override
-    public TokenList parse(File dir, String[] files) {
-        return this.parser.parse(dir, files);
+    public TokenList parse(File directory, String[] files) {
+        return this.parser.parse(directory, files);
     }
 
     @Override

jplag.frontend.java/src/main/java/de/jplag/java/Parser.java

@@ -1,6 +1,9 @@
 package de.jplag.java;
 
+import static java.util.stream.Collectors.toList;
+
 import java.io.File;
+import java.util.Arrays;
 
 import de.jplag.AbstractParser;
 import de.jplag.TokenList;
@@ -11,21 +14,17 @@ public class Parser extends AbstractParser {
     public TokenList parse(File directory, String[] files) {
         tokens = new TokenList();
         errors = 0;
-        File[] pathedFiles = new File[files.length];
-        for (int i = 0; i < files.length; i++) {
-            pathedFiles[i] = new File(directory, files[i]);
-        }
-        JavacAdapter javac = new JavacAdapter();
-        errors += javac.parseFiles(directory, pathedFiles, this);
+        var pathedFiles = Arrays.stream(files).map(it -> new File(directory, it)).collect(toList());
+        errors += new JavacAdapter().parseFiles(directory, pathedFiles, this);
         this.parseEnd();
         return tokens;
     }
 
-    public void add(int type, String filename, long line, long col, long length) {
-        tokens.addToken(new JavaToken(type, filename, (int) line, (int) col, (int) length));
+    public void add(int type, String filename, long line, long column, long length) {
+        tokens.addToken(new JavaToken(type, filename, (int) line, (int) column, (int) length));
     }
 
-    public void errorsInc() {
+    public void increaseErrors() {
         errors++;
     }
 }

jplag.frontend.java/src/main/java/de/jplag/java/TokenGeneratingTreeScanner.java

@@ -57,323 +57,341 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
         this.ast = ast;
     }
 
+    /**
+     * Convenience method that adds a specific token.
+     * @param tokenType is the type from {@link JavaTokenConstants}.
+     * @param position is the start position of the token.
+     * @param length is the length of the token.
+     */
+    private void addToken(int tokenType, long position, int length) {
+        parser.add(tokenType, filename, map.getLineNumber(position), map.getColumnNumber(position), length);
+    }
+
+    /**
+     * Convenience method that adds a specific token.
+     * @param tokenType is the type from {@link JavaTokenConstants}.
+     * @param start is the start position of the token.
+     * @param end is the end position of the token for the calculation of the length.
+     */
+    private void addToken(int tokenType, long start, long end) {
+        parser.add(tokenType, filename, map.getLineNumber(start), map.getColumnNumber(start), (end - start));
+    }
+
     @Override
     public Object visitBlock(BlockTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        long m = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_INIT_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 1);
-        Object ret = super.visitBlock(node, p);
-        parser.add(JavaTokenConstants.J_INIT_END, filename, map.getLineNumber(m - 1), map.getColumnNumber(m - 1), 1);
-        return ret;
+        long start = positions.getStartPosition(ast, node);
+        long end = positions.getEndPosition(ast, node) - 1;
+        addToken(JavaTokenConstants.J_INIT_BEGIN, start, 1);
+        Object result = super.visitBlock(node, p);
+        addToken(JavaTokenConstants.J_INIT_END, end, 1);
+        return result;
     }
 
     @Override
     public Object visitClass(ClassTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        long m = positions.getEndPosition(ast, node) - 1;
+        long start = positions.getStartPosition(ast, node);
+        long end = positions.getEndPosition(ast, node) - 1;
 
         if (node.getKind() == Tree.Kind.ENUM) {
-            parser.add(JavaTokenConstants.J_ENUM_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 4);
+            addToken(JavaTokenConstants.J_ENUM_BEGIN, start, 4);
         } else if (node.getKind() == Tree.Kind.INTERFACE) {
-            parser.add(JavaTokenConstants.J_INTERFACE_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 9);
+            addToken(JavaTokenConstants.J_INTERFACE_BEGIN, start, 9);
         } else if (node.getKind() == Tree.Kind.RECORD) {
-            parser.add(JavaTokenConstants.J_RECORD_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 1);
+            addToken(JavaTokenConstants.J_RECORD_BEGIN, start, 1);
         } else if (node.getKind() == Tree.Kind.ANNOTATION_TYPE) {
-            parser.add(JavaTokenConstants.J_ANNO_T_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 10);
+            addToken(JavaTokenConstants.J_ANNO_T_BEGIN, start, 10);
         } else if (node.getKind() == Tree.Kind.CLASS) {
-            parser.add(JavaTokenConstants.J_CLASS_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 5);
+            addToken(JavaTokenConstants.J_CLASS_BEGIN, start, 5);
         }
         Object result = super.visitClass(node, p);
         if (node.getKind() == Tree.Kind.ENUM) {
-            parser.add(JavaTokenConstants.J_ENUM_END, filename, map.getLineNumber(m), map.getColumnNumber(m), 1);
+            addToken(JavaTokenConstants.J_ENUM_END, end, 1);
         } else if (node.getKind() == Tree.Kind.INTERFACE) {
-            parser.add(JavaTokenConstants.J_INTERFACE_END, filename, map.getLineNumber(m), map.getColumnNumber(m), 1);
+            addToken(JavaTokenConstants.J_INTERFACE_END, end, 1);
         } else if (node.getKind() == Tree.Kind.RECORD) {
-            parser.add(JavaTokenConstants.J_RECORD_END, filename, map.getLineNumber(m), map.getColumnNumber(m), 1);
+            addToken(JavaTokenConstants.J_RECORD_END, end, 1);
         } else if (node.getKind() == Tree.Kind.ANNOTATION_TYPE) {
-            parser.add(JavaTokenConstants.J_ANNO_T_END, filename, map.getLineNumber(m), map.getColumnNumber(m), 1);
+            addToken(JavaTokenConstants.J_ANNO_T_END, end, 1);
         } else if (node.getKind() == Tree.Kind.CLASS) {
-            parser.add(JavaTokenConstants.J_CLASS_END, filename, map.getLineNumber(m), map.getColumnNumber(m), 1);
+            addToken(JavaTokenConstants.J_CLASS_END, end, 1);
         }
         return result;
     }
 
     @Override
     public Object visitImport(ImportTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_IMPORT, filename, map.getLineNumber(n), map.getColumnNumber(n), 6);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_IMPORT, start, 6);
         return super.visitImport(node, p);
     }
 
     @Override
     public Object visitPackage(PackageTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_PACKAGE, filename, map.getLineNumber(n), map.getColumnNumber(n), 7);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_PACKAGE, start, 7);
         return super.visitPackage(node, p);
     }
 
     @Override
     public Object visitMethod(MethodTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        long m = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_METHOD_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), node.getName().length());
-        Object ret = super.visitMethod(node, p);
-        parser.add(JavaTokenConstants.J_METHOD_END, filename, map.getLineNumber(m - 1), map.getColumnNumber(m - 1), 1);
-        return ret;
+        long start = positions.getStartPosition(ast, node);
+        long end = positions.getEndPosition(ast, node) - 1;
+        addToken(JavaTokenConstants.J_METHOD_BEGIN, start, node.getName().length());
+        Object result = super.visitMethod(node, p);
+        addToken(JavaTokenConstants.J_METHOD_END, end, 1);
+        return result;
     }
 
     @Override
     public Object visitSynchronized(SynchronizedTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        long m = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_SYNC_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 12);
-        Object ret = super.visitSynchronized(node, p);
-        parser.add(JavaTokenConstants.J_SYNC_END, filename, map.getLineNumber(m - 1), map.getColumnNumber(m - 1), 1);
-        return ret;
+        long start = positions.getStartPosition(ast, node);
+        long end = positions.getEndPosition(ast, node) - 1;
+        addToken(JavaTokenConstants.J_SYNC_BEGIN, start, 12);
+        Object result = super.visitSynchronized(node, p);
+        addToken(JavaTokenConstants.J_SYNC_END, end, 1);
+        return result;
     }
 
     @Override
     public Object visitDoWhileLoop(DoWhileLoopTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        long m = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_DO_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 2);
-        Object ret = super.visitDoWhileLoop(node, p);
-        parser.add(JavaTokenConstants.J_DO_END, filename, map.getLineNumber(m - 1), map.getColumnNumber(m - 1), 1);
-        return ret;
+        long start = positions.getStartPosition(ast, node);
+        long end = positions.getEndPosition(ast, node) - 1;
+        addToken(JavaTokenConstants.J_DO_BEGIN, start, 2);
+        Object result = super.visitDoWhileLoop(node, p);
+        addToken(JavaTokenConstants.J_DO_END, end, 1);
+        return result;
     }
 
     @Override
     public Object visitWhileLoop(WhileLoopTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        long m = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_WHILE_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 5);
-        Object ret = super.visitWhileLoop(node, p);
-        parser.add(JavaTokenConstants.J_WHILE_END, filename, map.getLineNumber(m - 1), map.getColumnNumber(m - 1), 1);
-        return ret;
+        long start = positions.getStartPosition(ast, node);
+        long end = positions.getEndPosition(ast, node) - 1;
+        addToken(JavaTokenConstants.J_WHILE_BEGIN, start, 5);
+        Object result = super.visitWhileLoop(node, p);
+        addToken(JavaTokenConstants.J_WHILE_END, end, 1);
+        return result;
     }
 
     @Override
     public Object visitForLoop(ForLoopTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        long m = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_FOR_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 3);
-        Object ret = super.visitForLoop(node, p);
-        parser.add(JavaTokenConstants.J_FOR_END, filename, map.getLineNumber(m - 1), map.getColumnNumber(m - 1), 1);
-        return ret;
+        long start = positions.getStartPosition(ast, node);
+        long end = positions.getEndPosition(ast, node) - 1;
+        addToken(JavaTokenConstants.J_FOR_BEGIN, start, 3);
+        Object result = super.visitForLoop(node, p);
+        addToken(JavaTokenConstants.J_FOR_END, end, 1);
+        return result;
     }
 
     @Override
     public Object visitEnhancedForLoop(EnhancedForLoopTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        long m = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_FOR_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 3);
-        Object ret = super.visitEnhancedForLoop(node, p);
-        parser.add(JavaTokenConstants.J_FOR_END, filename, map.getLineNumber(m - 1), map.getColumnNumber(m - 1), 1);
-        return ret;
+        long start = positions.getStartPosition(ast, node);
+        long end = positions.getEndPosition(ast, node) - 1;
+        addToken(JavaTokenConstants.J_FOR_BEGIN, start, 3);
+        Object result = super.visitEnhancedForLoop(node, p);
+        addToken(JavaTokenConstants.J_FOR_END, end, 1);
+        return result;
     }
 
     @Override
     public Object visitSwitch(SwitchTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        long m = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_SWITCH_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 6);
-        Object ret = super.visitSwitch(node, p);
-        parser.add(JavaTokenConstants.J_SWITCH_END, filename, map.getLineNumber(m - 1), map.getColumnNumber(m - 1), 1);
-        return ret;
+        long start = positions.getStartPosition(ast, node);
+        long end = positions.getEndPosition(ast, node) - 1;
+        addToken(JavaTokenConstants.J_SWITCH_BEGIN, start, 6);
+        Object result = super.visitSwitch(node, p);
+        addToken(JavaTokenConstants.J_SWITCH_END, end, 1);
+        return result;
     }
 
     @Override
     public Object visitSwitchExpression(SwitchExpressionTree node, Object parameterValue) {
         long start = positions.getStartPosition(ast, node);
-        long end = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_SWITCH_BEGIN, filename, map.getLineNumber(start), map.getColumnNumber(start), 6);
+        long end = positions.getEndPosition(ast, node) - 1;
+        addToken(JavaTokenConstants.J_SWITCH_BEGIN, start, 6);
         Object result = super.visitSwitchExpression(node, parameterValue);
-        parser.add(JavaTokenConstants.J_SWITCH_END, filename, map.getLineNumber(end - 1), map.getColumnNumber(end - 1), 1);
+        addToken(JavaTokenConstants.J_SWITCH_END, end, 1);
         return result;
     }
 
     @Override
     public Object visitCase(CaseTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_CASE, filename, map.getLineNumber(n), map.getColumnNumber(n), 4);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_CASE, start, 4);
         return super.visitCase(node, p);
     }
 
     @Override
     public Object visitTry(TryTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
+        long start = positions.getStartPosition(ast, node);
         if (node.getResources().isEmpty())
-            parser.add(JavaTokenConstants.J_TRY_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 3);
+            addToken(JavaTokenConstants.J_TRY_BEGIN, start, 3);
         else
-            parser.add(JavaTokenConstants.J_TRY_WITH_RESOURCE, filename, map.getLineNumber(n), map.getColumnNumber(n), 3);
+            addToken(JavaTokenConstants.J_TRY_WITH_RESOURCE, start, 3);
         if (node.getFinallyBlock() != null)
-            parser.add(JavaTokenConstants.J_FINALLY, filename, map.getLineNumber(n), map.getColumnNumber(n), 3);
+            addToken(JavaTokenConstants.J_FINALLY, start, 3);
         return super.visitTry(node, p);
     }
 
     @Override
     public Object visitCatch(CatchTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        long m = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_CATCH_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 5);
-        Object ret = super.visitCatch(node, p);
-        parser.add(JavaTokenConstants.J_CATCH_END, filename, map.getLineNumber(m - 1), map.getColumnNumber(m - 1), 1);
-        return ret;
+        long start = positions.getStartPosition(ast, node);
+        long end = positions.getEndPosition(ast, node) - 1;
+        addToken(JavaTokenConstants.J_CATCH_BEGIN, start, 5);
+        Object result = super.visitCatch(node, p);
+        addToken(JavaTokenConstants.J_CATCH_END, end, 1);
+        return result;
     }
 
     @Override
     public Object visitIf(IfTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        long m = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_IF_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 2);
+        long start = positions.getStartPosition(ast, node);
+        long end = positions.getEndPosition(ast, node) - 1;
+        addToken(JavaTokenConstants.J_IF_BEGIN, start, 2);
         node.getCondition().accept(this, p);
         node.getThenStatement().accept(this, p);
         if (node.getElseStatement() != null) {
-            n = positions.getStartPosition(ast, node.getElseStatement());
-            parser.add(JavaTokenConstants.J_ELSE, filename, map.getLineNumber(n), map.getColumnNumber(n), 4);
+            start = positions.getStartPosition(ast, node.getElseStatement());
+            addToken(JavaTokenConstants.J_ELSE, start, 4);
             node.getElseStatement().accept(this, p);
         }
-        parser.add(JavaTokenConstants.J_IF_END, filename, map.getLineNumber(m - 1), map.getColumnNumber(m - 1), 1);
+        addToken(JavaTokenConstants.J_IF_END, end, 1);
         return null;
     }
 
     @Override
     public Object visitBreak(BreakTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_BREAK, filename, map.getLineNumber(n), map.getColumnNumber(n), 5);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_BREAK, start, 5);
         return super.visitBreak(node, p);
     }
 
     @Override
     public Object visitContinue(ContinueTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_CONTINUE, filename, map.getLineNumber(n), map.getColumnNumber(n), 8);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_CONTINUE, start, 8);
         return super.visitContinue(node, p);
     }
 
     @Override
     public Object visitReturn(ReturnTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_RETURN, filename, map.getLineNumber(n), map.getColumnNumber(n), 6);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_RETURN, start, 6);
         return super.visitReturn(node, p);
     }
 
     @Override
     public Object visitThrow(ThrowTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_THROW, filename, map.getLineNumber(n), map.getColumnNumber(n), 5);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_THROW, start, 5);
         return super.visitThrow(node, p);
     }
 
     @Override
     public Object visitNewClass(NewClassTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
+        long start = positions.getStartPosition(ast, node);
         if (node.getTypeArguments().size() > 0) {
-            parser.add(JavaTokenConstants.J_GENERIC, filename, map.getLineNumber(n), map.getColumnNumber(n),
-                    3 + node.getIdentifier().toString().length());
+            addToken(JavaTokenConstants.J_GENERIC, start, 3 + node.getIdentifier().toString().length());
         }
-        parser.add(JavaTokenConstants.J_NEWCLASS, filename, map.getLineNumber(n), map.getColumnNumber(n), 3);
+        addToken(JavaTokenConstants.J_NEWCLASS, start, 3);
         return super.visitNewClass(node, p);
     }
 
     @Override
     public Object visitTypeParameter(TypeParameterTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
+        long start = positions.getStartPosition(ast, node);
         // This is odd, but also done like this in Java17
-        parser.add(JavaTokenConstants.J_GENERIC, filename, map.getLineNumber(n), map.getColumnNumber(n), 1);
+        addToken(JavaTokenConstants.J_GENERIC, start, 1);
         return super.visitTypeParameter(node, p);
     }
 
     @Override
     public Object visitNewArray(NewArrayTree node, Object arg1) {
-        long n = positions.getStartPosition(ast, node);
-        long m = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_NEWARRAY, filename, map.getLineNumber(n), map.getColumnNumber(n), 3);
+        long start = positions.getStartPosition(ast, node);
+        long end = positions.getEndPosition(ast, node) - 1;
+        addToken(JavaTokenConstants.J_NEWARRAY, start, 3);
         if (node.getInitializers() != null && !node.getInitializers().isEmpty()) {
-            n = positions.getStartPosition(ast, node.getInitializers().get(0));
-            parser.add(JavaTokenConstants.J_ARRAY_INIT_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 1);
-            parser.add(JavaTokenConstants.J_ARRAY_INIT_END, filename, map.getLineNumber(m - 1), map.getColumnNumber(m - 1), 1);
+            start = positions.getStartPosition(ast, node.getInitializers().get(0));
+            addToken(JavaTokenConstants.J_ARRAY_INIT_BEGIN, start, 1);
+            addToken(JavaTokenConstants.J_ARRAY_INIT_END, end, 1);
         }
         return super.visitNewArray(node, arg1);
     }
 
     @Override
     public Object visitAssignment(AssignmentTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_ASSIGN, filename, map.getLineNumber(n), map.getColumnNumber(n), 1);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_ASSIGN, start, 1);
         return super.visitAssignment(node, p);
     }
 
     @Override
     public Object visitAssert(AssertTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_ASSERT, filename, map.getLineNumber(n), map.getColumnNumber(n), 6);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_ASSERT, start, 6);
         return super.visitAssert(node, p);
     }
 
     @Override
     public Object visitVariable(VariableTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_VARDEF, filename, map.getLineNumber(n), map.getColumnNumber(n), node.toString().length());
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_VARDEF, start, node.toString().length());
         return super.visitVariable(node, p);
     }
 
     @Override
     public Object visitConditionalExpression(ConditionalExpressionTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_COND, filename, map.getLineNumber(n), map.getColumnNumber(n), 1);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_COND, start, 1);
         return super.visitConditionalExpression(node, p);
     }
 
     @Override
     public Object visitMethodInvocation(MethodInvocationTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_APPLY, filename, map.getLineNumber(n), map.getColumnNumber(n),
-                positions.getEndPosition(ast, node.getMethodSelect()) - n);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_APPLY, start, positions.getEndPosition(ast, node.getMethodSelect()) - start);
         return super.visitMethodInvocation(node, p);
     }
 
     @Override
     public Object visitAnnotation(AnnotationTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_ANNO, filename, map.getLineNumber(n), map.getColumnNumber(n), 1);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_ANNO, start, 1);
         return super.visitAnnotation(node, p);
     }
 
     @Override
     public Object visitModule(ModuleTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        long m = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_MODULE_BEGIN, filename, map.getLineNumber(n), map.getColumnNumber(n), 6);
-        Object ret = super.visitModule(node, p);
-        parser.add(JavaTokenConstants.J_MODULE_END, filename, map.getLineNumber(m - 1), map.getColumnNumber(m - 1), 1);
-        return ret;
+        long start = positions.getStartPosition(ast, node);
+        long end = positions.getEndPosition(ast, node) - 1;
+        addToken(JavaTokenConstants.J_MODULE_BEGIN, start, 6);
+        Object result = super.visitModule(node, p);
+        addToken(JavaTokenConstants.J_MODULE_END, end, 1);
+        return result;
     }
 
     @Override
     public Object visitRequires(RequiresTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_REQUIRES, filename, map.getLineNumber(n), map.getColumnNumber(n), 8);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_REQUIRES, start, 8);
         return super.visitRequires(node, p);
     }
 
     @Override
     public Object visitProvides(ProvidesTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_PROVIDES, filename, map.getLineNumber(n), map.getColumnNumber(n), 8);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_PROVIDES, start, 8);
         return super.visitProvides(node, p);
     }
 
     @Override
     public Object visitExports(ExportsTree node, Object p) {
-        long n = positions.getStartPosition(ast, node);
-        parser.add(JavaTokenConstants.J_EXPORTS, filename, map.getLineNumber(n), map.getColumnNumber(n), 7);
+        long start = positions.getStartPosition(ast, node);
+        addToken(JavaTokenConstants.J_EXPORTS, start, 7);
         return super.visitExports(node, p);
     }
 
     @Override
     public Object visitErroneous(ErroneousTree node, Object p) {
-        parser.errorsInc();
+        parser.increaseErrors();
         return super.visitErroneous(node, p);
     }
 
@@ -381,7 +399,7 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitYield(YieldTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_YIELD, filename, map.getLineNumber(start), map.getColumnNumber(start), (end - start));
+        addToken(JavaTokenConstants.J_YIELD, start, end);
         return super.visitYield(node, p);
     }
 
@@ -389,7 +407,7 @@ final class TokenGeneratingTreeScanner extends TreeScanner<Object, Object> {
     public Object visitDefaultCaseLabel(DefaultCaseLabelTree node, Object p) {
         long start = positions.getStartPosition(ast, node);
         long end = positions.getEndPosition(ast, node);
-        parser.add(JavaTokenConstants.J_DEFAULT, filename, map.getLineNumber(start), map.getColumnNumber(start), (end - start));
+        addToken(JavaTokenConstants.J_DEFAULT, start, end);
         return super.visitDefaultCaseLabel(node, p);
     }
 }

jplag.frontend.python-3/src/main/java/de/jplag/python3/Language.java

@@ -11,7 +11,7 @@ public class Language implements de.jplag.Language {
 
     public Language(ErrorConsumer program) {
         this.parser = new Parser();
-        this.parser.setProgram(program);
+        this.parser.setErrorConsumer(program);
     }
 
     @Override

jplag.frontend.scheme/src/main/java/de/jplag/scheme/Language.java

@@ -9,7 +9,7 @@ public class Language implements de.jplag.Language {
 
     public Language(ErrorConsumer program) {
         this.parser = new Parser();
-        this.parser.setProgram(program);
+        this.parser.setErrorConsumer(program);
 
     }
 

jplag.frontend.text/src/main/java/de/jplag/text/Language.java

@@ -11,7 +11,7 @@ public class Language implements de.jplag.Language {
     private final Parser parser = new Parser();
 
     public Language(ErrorConsumer program) {
-        this.parser.setProgram(program);
+        this.parser.setErrorConsumer(program);
     }
 
     @Override