Add complete sample file, refined Token extraction

jplag.frontend.scala/src/main/scala/de/jplag/scala/Parser.scala

@@ -29,10 +29,12 @@ class Parser(consumer: ErrorConsumer) extends AbstractParser(consumer) {
         }
 
         private def processCases(cases: List[Case]) : Unit = cases.foreach {
-            c =>
-                add(ScalaTokenConstants.CaseBegin, c, fromEnd=false)
-                apply(c)
-                add(ScalaTokenConstants.CaseEnd, c, fromEnd=true)
+            case c@Case(pattern, condition, body) =>
+                add(ScalaTokenConstants.CaseStatement, c, fromEnd = false)
+                apply(pattern)
+                apply(condition)
+                enclose(body, TR(Some(CaseBegin), Some(CaseEnd)))
+            case _ =>
         }
 
         def isNotArithmetic(op: Term.Name) : Boolean = {
@@ -41,11 +43,20 @@ class Parser(consumer: ErrorConsumer) extends AbstractParser(consumer) {
 
         private def doMatch(tree: Tree): TR = {
             tree match {
-                case Term.Do(_) => TR(Some(DoBegin), Some(DoEnd))
+                case Term.Do(body, condition) => TR(Some(DoWhile), Some(DoWhileEnd), traverse = _ => {
+                    enclose(body, TR(Some(DoBodyBegin), Some(DoBodyEnd)))
+                    apply(condition)
+                })
                 case Term.Assign(_) => TR(Some(Assign), None)
-                case Term.While(_) => TR(Some(WhileBegin), Some(WhileEnd))
-                case Term.For(_) =>
-                    TR(Some(ForBegin), Some(ForEnd))
+                case Term.While(condition, body) => TR(Some(While), traverse = _ => {
+                    apply(condition)
+                    enclose(body, TR(Some(WhileBodyBegin), Some(WhileBodyEnd)))
+                })
+                case Term.For(enumerators, body) =>
+                    TR(Some(For), traverse = _ => {
+                        apply(enumerators)
+                        enclose(body, TR(Some(ForBodyBegin), Some(ForBodyEnd)))
+                    })
                 case Term.Try(expr, catchExpr, finallyExpr) => TR(Some(TryBegin),
                     traverse = _ => {
                         apply(expr)
@@ -64,9 +75,7 @@ class Parser(consumer: ErrorConsumer) extends AbstractParser(consumer) {
                 case Term.TryWithHandler(expr, catchp, finallyp) => TR(Some(TryBegin),
                     traverse = _ => {
                         apply(expr)
-                        add(CatchBegin, catchp, fromEnd=false)
-                        apply(catchp)
-                        add(CatchEnd, catchp, fromEnd = true)
+                        enclose(catchp, TR(Some(CatchBegin), Some(CatchEnd)))
 
                         maybeAddAndApply(finallyp, ScalaTokenConstants.Finally)
                     })
@@ -86,7 +95,7 @@ class Parser(consumer: ErrorConsumer) extends AbstractParser(consumer) {
                             }
                         }
                     })
-                case Term.NewAnonymous(_) => TR(Some(NewCreationBegin), Some(ScalaTokenConstants.NewCreationBegin))
+                case Term.NewAnonymous(_) => TR(Some(NewCreationBegin), Some(NewCreationEnd))
                 case Term.Return(_) => TR(Some(ScalaTokenConstants.Return))
                 case Term.Match(expr, cases) => TR(traverse = _ => {
                     add(MatchBegin, tree, fromEnd=false)
@@ -102,20 +111,18 @@ class Parser(consumer: ErrorConsumer) extends AbstractParser(consumer) {
                     add(PartialFunctionEnd, tree, fromEnd=true)
                 })
                 case Term.ForYield(_) => TR(traverse = _ => {
-                    add(ForBegin, tree, fromEnd=false)
+                    add(ForBodyBegin, tree, fromEnd=false)
                     add(Yield, tree, fromEnd = false)
 
                     tree.children.foreach(apply)
 
-                    add(ForEnd, tree, fromEnd = true)
+                    add(ForBodyEnd, tree, fromEnd = true)
                 })
                 case Term.If(conditionExpr, thenExpression, elseExpression) => TR(traverse = _ => {
                     add(If, tree, fromEnd=false)
                     apply(conditionExpr)
-                    add(IfBegin, thenExpression, fromEnd=false)
 
-                    apply(thenExpression)
-                    add(IfEnd, thenExpression, fromEnd=true)
+                    enclose(thenExpression, TR(Some(IfBegin), Some(IfEnd)))
 
                     elseExpression match {
                         case Lit.Unit() => apply(elseExpression)
@@ -123,27 +130,31 @@ class Parser(consumer: ErrorConsumer) extends AbstractParser(consumer) {
                             val elseStart = tree.pos.text.indexOf("else", thenExpression.pos.end - tree.pos.start)
                             val elsePosition = Position.Range(tree.pos.input, tree.pos.start + elseStart, tree.pos.start + elseStart + 4)
                             add(Else, elsePosition.startLine + 1, elsePosition.startColumn + 1, elsePosition.text.length)
-                            add(ElseBegin, elseExpression, fromEnd = false)
-                            apply(elseExpression)
-                            add(ElseEnd, elseExpression, fromEnd = true)
+                            enclose(elseExpression, TR(Some(ElseBegin), Some(ElseEnd)))
                     }
                 })
 
                 case scala.meta.Pkg(_) => TR(Some(Package))
                 case scala.meta.Import(_) => TR(Some(ScalaTokenConstants.Import))
 
-                case Defn.Def(_, name, typeParams, paramss, _, body) =>
+                case Defn.Def(mods, name, typeParams, paramss, _, body) =>
                     TR(traverse = _ => {
+                        applyRecursively(mods)
                         add(MethodDef, name, fromEnd = false)
                         for (tParam <- typeParams) add(TypeParameter, tParam, fromEnd = false)
                         for (params <- paramss)
                             for (param <- params) add(Parameter, param, fromEnd = false)
-                        add(MethodBegin, body, fromEnd = false)
-                        apply(body)
-                        add(MethodEnd, body, fromEnd = true)
+
+                        enclose(body, TR(Some(MethodBegin), Some(MethodEnd)))
                     })
-                case Defn.Macro(_) => TR(Some(MacroBegin), Some(MacroEnd))
-                case Defn.Class(_) => TR(Some(ClassBegin), Some(ClassEnd))
+                case Defn.Macro(mods, name, tparams, paramss, declType, body) => TR(Some(Macro), traverse = _ => {
+                    applyRecursively(mods)
+                    applyRecursively(tparams)
+                    applyRecursively(paramss)
+                    enclose(body, TR(Some(MacroBegin), Some(MacroEnd)))
+                })
+                case Defn.Class(_) =>
+                    TR(Some(ClassBegin), Some(ClassEnd))
                 case Defn.Object(_) => TR(Some(ObjectBegin), Some(ObjectEnd))
                 case Defn.Trait(_) => TR(Some(TraitBegin), Some(TraitEnd))
                 case Defn.Type(_) => TR(Some(ScalaTokenConstants.Type))
@@ -176,12 +187,20 @@ class Parser(consumer: ErrorConsumer) extends AbstractParser(consumer) {
                 })
 
                 case Decl.Var(_) => TR(Some(VariableDefinition))
-                case Decl.Var(_) => TR(Some(VariableDefinition))
+                case Decl.Val(_) => TR(Some(VariableDefinition))
                 case Decl.Def(_) => TR(Some(MethodBegin), Some(MethodEnd))
                 case Decl.Type(_) => TR(Some(ScalaTokenConstants.Type))
 
-                case Ctor.Secondary(_) => TR(Some(ConstructorBegin), Some(ConstructorEnd))
-
+                case Ctor.Secondary(_) =>
+                    TR(Some(ConstructorBegin), Some(ConstructorEnd))
+                case Init(tpe, name, argss) if argss.nonEmpty => TR(traverse = _ => {
+                    for (args <- argss) {
+                        for (arg <- args) {
+                            add(Argument, arg, fromEnd = false)
+                            apply(arg)
+                        }
+                    }
+                })
                 case Enumerator.Guard(_) => TR(Some(Guard))
 
                 case Term.Param(_) => TR(traverse = _ => add(Parameter, tree, fromEnd = false))
@@ -211,24 +230,45 @@ class Parser(consumer: ErrorConsumer) extends AbstractParser(consumer) {
                 })
                 case Term.New(_) => TR(Some(NewObject))
                 case Self(_) => TR(Some(SelfType))
-                case Term.Block(_) => TR(Some(BlockStart), Some(BlockEnd))
+                case block@Term.Block(_) if block.parent.nonEmpty && block.parent.get.isInstanceOf[Term.Apply] =>
+                    TR(Some(BlockStart), Some(BlockEnd))
                 case Enumerator.Generator(_) => TR(Some(EnumGenerator))
                 case meta.Type.Param(_) => TR(Some(TypeParameter))
+
                 case _ => TR()
             }
         }
 
+        def applyRecursively[T](els: List[T]): Unit = els.foreach( {
+             case tree: Tree => apply(tree)
+             case treeList: List[_] => applyRecursively(treeList)
+        })
+
         override def apply(tree: Tree): Unit = {
-            val res = doMatch(tree)
+            val record : TR = doMatch(tree)
+
+            record.before match {
+                case Some(value) => add(value, tree, fromEnd = false)
+                case None =>
+            }
+
+            record.traverse(tree)
+
+            record.after match {
+                case Some(value) => add(value, tree, fromEnd = true)
+                case None =>
+            }
+        }
 
-            res.before match {
+        def enclose(tree: Tree, record: TR): Unit = {
+            record.before match {
                 case Some(value) => add(value, tree, fromEnd = false)
                 case None =>
             }
 
-            res.traverse(tree)
+            apply(tree)
 
-            res.after match {
+            record.after match {
                 case Some(value) => add(value, tree, fromEnd = true)
                 case None =>
             }

jplag.frontend.scala/src/main/scala/de/jplag/scala/ScalaTokenConstants.scala

@@ -1,9 +1,10 @@
 package de.jplag.scala
 
-import de.jplag.{Token, TokenConstants}
-
-import scala.meta.inputs.Position
+import de.jplag.TokenConstants
 
+/**
+ * This Java-compatible enumeration stores the possible types of tokens.
+ */
 object ScalaTokenConstants extends Enumeration with TokenConstants {
 
   val FileEnd: Value = Value(TokenConstants.FILE_END, "<EOF>")
@@ -16,12 +17,17 @@ object ScalaTokenConstants extends Enumeration with TokenConstants {
   val MethodBegin: Value = Value("METHOD{")
   val MethodEnd: Value = Value("}METHOD")
   val VariableDefinition: Value = Value("VAR_DEF")
-  val DoBegin: Value = Value("DO{")
-  val DoEnd: Value = Value("}DO")
-  val WhileBegin: Value = Value("WHILE{")
-  val WhileEnd: Value = Value("}WHILE")
-  val ForBegin: Value = Value("FOR{")
-  val ForEnd: Value = Value("}FOR")
+  val DoWhile: Value = Value("DO-WHILE")
+  val DoWhileEnd: Value = Value("END-DO-WHILE")
+  val DoBodyBegin: Value = Value("DO{")
+  val DoBodyEnd: Value = Value("}DO")
+  val While: Value = Value("WHILE")
+  val WhileBodyBegin: Value = Value("WHILE{")
+  val WhileBodyEnd: Value = Value("}WHILE")
+  val For: Value = Value("FOR")
+  val ForBodyBegin: Value = Value("FOR{")
+  val ForBodyEnd: Value = Value("}FOR")
+  val CaseStatement: Value = Value("CASE")
   val CaseBegin: Value = Value("CASE{")
   val CaseEnd: Value = Value("}CASE")
   val TryBegin: Value = Value("TRY{")
@@ -44,23 +50,12 @@ object ScalaTokenConstants extends Enumeration with TokenConstants {
   val TraitEnd: Value = Value("}TRAIT")
   val ConstructorBegin: Value = Value("CONSTR{")
   val ConstructorEnd: Value = Value("}CONSTR")
-  val InitBegin: Value = Value("INIT{")
-  val InitEnd: Value = Value("}INIT")
-  val ArrayInitBegin: Value = Value("ARRAY(){")
-  val ArrayInitEnd: Value = Value("}ARRAY")
   val MatchBegin: Value = Value("MATCH{")
   val MatchEnd: Value = Value("}MATCH")
   val Guard: Value = Value("GUARD")
-  val Anno: Value = Value("ANNO")
-  val AnnoMarker: Value = Value("ANNO_MARKER")
-  val AnnoMBegin: Value = Value("ANNO_M{")
-  val AnnoMEnd: Value = Value("}ANNO_M")
-  val AnnoTBegin: Value = Value("ANNO_T{")
-  val AnnoTEnd: Value = Value("}ANNO_T")
-  val AnnoCBegin: Value = Value("ANNO_C{")
-  val AnnoCEnd: Value = Value("}ANNO_C")
   val ObjectBegin: Value = Value("OBJECT{")
   val ObjectEnd: Value = Value("}OBJECT")
+  val Macro: Value = Value("MACRO")
   val MacroBegin: Value = Value("MACRO{")
   val MacroEnd: Value = Value("}MACRO")
   val Type: Value = Value("TYPE")
@@ -82,5 +77,6 @@ object ScalaTokenConstants extends Enumeration with TokenConstants {
   val BlockEnd: Value = Value("}")
   val EnumGenerator: Value = Value("ENUMERATE")
   val Member: Value = Value("MEMBER")
+
 }
 

jplag.frontend.scala/src/test/java/de/jplag/scala/ScalaFrontendTest.java

@@ -28,7 +28,7 @@ class ScalaFrontendTest {
     /**
      * Test source file that is supposed to produce a complete set of tokens, i.e. all types of tokens.
      */
-    private static final String COMPLETE_TEST_FILE = "Typers.scala";
+    private static final String COMPLETE_TEST_FILE = "Complete.scala";
 
     /**
      * Regular expression that describes lines consisting only of whitespace and optionally a line comment.
@@ -48,7 +48,7 @@ class ScalaFrontendTest {
     private static final int NOT_SET = -1;
 
     private final Logger logger = LoggerFactory.getLogger("Scala frontend test");
-    private final String[] testFiles = new String[] {"Typers.scala", "Parser.scala"};
+    private final String[] testFiles = new String[] {"Complete.scala", "Parser.scala"};
     private final File testFileLocation = Path.of("src", "test", "resources", "de", "jplag", "scala").toFile();
     private Language language;
 
@@ -107,14 +107,18 @@ class ScalaFrontendTest {
 
             // Keep only lines that have no tokens
             codeLines.removeAll(tokenLines);
-            System.out.println(codeLines);
-
-            /*
-             * if (codeLines.length > tokenLines.length) { var diffLine = IntStream.range(0, codeLines.length) .dropWhile(lineIdx ->
-             * lineIdx < tokenLines.length && codeLines[lineIdx] == tokenLines[lineIdx]).findFirst(); diffLine.ifPresent( lineIdx ->
-             * fail("Line %d of file '%s' is not represented in the token list.".formatted(codeLines[lineIdx], fileName))); }
-             * assertArrayEquals(codeLines, tokenLines);
-             */
+
+            double coverage = 1.d - (codeLines.size() * 1.d / (codeLines.size() + tokenLines.size()));
+            if (coverage == 1) {
+                logger.info("All lines covered.");
+            } else {
+                logger.info("Coverage: %.1f%%.".formatted(coverage * 100));
+                logger.info("Missing lines {}", codeLines);
+                if (coverage < 0.9) {
+                    fail("Source coverage is unsatisfactory");
+                }
+            }
+
         } catch (IOException exception) {
             logger.info("Error while reading test file %s".formatted(fileName), exception);
             fail();

jplag.frontend.scala/src/test/resources/de/jplag/scala/Complete.scala

@@ -0,0 +1,117 @@
+package mypackage
+
+import scala.annotation.tailrec
+import scala.otherpackage
+
+class Complete extends SuperClass with MyTrait {
+
+    val constantMember = 1
+
+    def method(arg: ArgType): Unit = {
+
+        var variable : Int = 2
+
+        do method() while (false)
+        while (false) method()
+
+        for (index <- 0 to 10 by 1 if isPrime(index)) {
+            method(index)
+            (index % 2) match {
+                case 0 => "even"
+                case 1 => "odd"
+            }
+        }
+
+        if (true) {
+
+        } else {
+
+        }
+
+        try {
+            method()
+            throw new Exception()
+        } catch {
+            case exception : Exception if true =>
+            case _ =>
+        } finally {
+
+        }
+
+        val anonymousClassObject = new AnyVal {}
+
+    }
+
+    def getOne() : Int = {
+        return 1
+    }
+
+    object NumberGenerator {
+
+        def getNumber(i : Int): Unit = {
+            if (i <= 0) i else 1 + getNumber(i - 1)
+        }
+
+    }
+
+    class Person(var firstName: String, var lastName: String) {
+
+        def this(var firstName: String) = {
+            this(firstName, "Smith")
+        }
+    }
+
+    def create(): Unit = {
+        val obj = new Person("Mickey", "Mouse")
+    }
+
+    def applyPartialFunction(strings: Seq[String]) =
+        strings.collect({
+            case s if s.length > 5 => s
+            case _ =>
+        })
+
+    def applyTotalFunction(strings: Seq[String]) =
+        strings.find(s => {
+            s.substring(0,3).equals("abc")
+        })
+
+    def callMacro(msg: Any) = macro macroImpl
+
+    def macroImpl(c: Context)(msg: c.Expr[Any]) = println(msg)
+
+    def yieldingMethod(list: Seq[String]) = {
+        for (string <- list if string.startsWith("_")) yield string.substring(1)
+    }
+
+    trait MyTrait[U] {
+        self =>
+        type T <: U
+        val member: T
+    }
+
+    class MyTraitImpl extends MyTrait[Complete] {}
+
+    def reverse[T](list: Seq[T]): Seq[T] = {
+        if (list.isEmpty) list
+        else reverse(list.tail) :: list.head
+    }
+
+    def useReverse(): Unit = {
+        val list = Array[Int](3, 2, 1)
+        reverse[Int](list)
+    }
+
+    def innerBlock(): Unit = {
+        val a = 1
+        {
+            val b = 2
+        }
+        println("b is not accessible here.")
+    }
+    /*
+    Enums are a feature of Scala3, not supported by the parser.
+    enum Menu:
+        case PizzaMargharita, SpaghettiBolognese, Fries
+    */
+}

jplag.frontend.scala/src/test/resources/de/jplag/scala/Parser.scala

@@ -8,267 +8,323 @@ import scala.meta._
 
 
 class Parser(consumer: ErrorConsumer) extends AbstractParser(consumer) {
-  private var currentFile : String = _
+    private var currentFile : String = _
 
-  var tokens : TokenList = _
+    var tokens : TokenList = _
 
-  private var lastInMethod: List[Boolean] = List[Boolean]()
+    private var lastInMethod: List[Boolean] = List[Boolean]()
 
-  private val traverser: Traverser = new Traverser {
+    private val traverser: Traverser = new Traverser {
 
-    case class TR(before: Option[ScalaTokenConstants.Value] = None,
-                  after: Option[ScalaTokenConstants.Value] = None,
-                  traverse: Tree => Unit = _.children.foreach(traverser.apply)
-                 )
+        case class TR(before: Option[ScalaTokenConstants.Value] = None,
+                      after: Option[ScalaTokenConstants.Value] = None,
+                      traverse: Tree => Unit = _.children.foreach(traverser.apply)
+                     )
 
-    private def maybeAddAndApply(tree: Option[Tree], token: ScalaTokenConstants.Value): Unit = tree match {
-      case Some(exp) =>
-        add(token, exp, fromEnd = false)
-        apply(exp)
-      case None =>
-    }
-
-    private def processCases(cases: List[Case]) : Unit = cases.foreach {
-      c =>
-        add(ScalaTokenConstants.CaseBegin, c, fromEnd=false)
-        apply(c)
-        add(ScalaTokenConstants.CaseEnd, c, fromEnd=true)
-    }
+        private def maybeAddAndApply(tree: Option[Tree], token: ScalaTokenConstants.Value): Unit = tree match {
+            case Some(exp) =>
+                add(token, exp, fromEnd = false)
+                apply(exp)
+            case None =>
+        }
 
-    def isNotArithmetic(op: Term.Name) = {
-      op.value.matches("""[_\w]+""")
-    }
-
-    private def doMatch(tree: Tree): TR = {
-      tree match {
-        case Term.Do(_) => TR(Some(DoBegin), Some(DoEnd))
-        case Term.Assign(_) => TR(Some(Assign), None)
-        case Term.While(_) => TR(Some(WhileBegin), Some(WhileEnd))
-        case Term.For(_) =>
-          TR(Some(ForBegin), Some(ForEnd))
-        case Term.Try(expr, catchExpr, finallyExpr) => TR(Some(TryBegin),
-          traverse = _ => {
-            apply(expr)
-            if (catchExpr.nonEmpty) {
-              val start = catchExpr.head.pos
-              val end = catchExpr.last.pos
-              val len = end.endLine - start.startLine
-
-              add(CatchBegin, start.startLine, start.startColumn, len)
-              processCases(catchExpr)
-              add(CatchEnd, end.endLine, end.endColumn, len)
+        private def processCases(cases: List[Case]) : Unit = cases.foreach {
+            case c@Case(pattern, condition, body) =>
+                add(ScalaTokenConstants.CaseStatement, c, fromEnd = false)
+                apply(pattern)
+                apply(condition)
+                enclose(body, TR(Some(CaseBegin), Some(CaseEnd)))
+            case _ =>
+        }
+
+        def isNotArithmetic(op: Term.Name) : Boolean = {
+            op.value.matches("""[_\w]+""")
+        }
+
+        private def doMatch(tree: Tree): TR = {
+            tree match {
+                case Term.Do(body, condition) => TR(Some(DoWhile), Some(DoWhileEnd), traverse = _ => {
+                    enclose(body, TR(Some(DoBodyBegin), Some(DoBodyEnd)))
+                    apply(condition)
+                })
+                case Term.Assign(_) => TR(Some(Assign), None)
+                case Term.While(condition, body) => TR(Some(While), traverse = _ => {
+                    apply(condition)
+                    enclose(body, TR(Some(WhileBodyBegin), Some(WhileBodyEnd)))
+                })
+                case Term.For(enumerators, body) =>
+                    TR(Some(For), traverse = _ => {
+                        apply(enumerators)
+                        enclose(body, TR(Some(ForBodyBegin), Some(ForBodyEnd)))
+                    })
+                case Term.Try(expr, catchExpr, finallyExpr) => TR(Some(TryBegin),
+                    traverse = _ => {
+                        apply(expr)
+                        if (catchExpr.nonEmpty) {
+                            val start = catchExpr.head.pos
+                            val end = catchExpr.last.pos
+                            val len = end.endLine - start.startLine
+
+                            add(CatchBegin, start.startLine, start.startColumn, len)
+                            processCases(catchExpr)
+                            add(CatchEnd, end.endLine, end.endColumn, len)
+                        }
+
+                        maybeAddAndApply(finallyExpr, ScalaTokenConstants.Finally)
+                    })
+                case Term.TryWithHandler(expr, catchp, finallyp) => TR(Some(TryBegin),
+                    traverse = _ => {
+                        apply(expr)
+                        enclose(catchp, TR(Some(CatchBegin), Some(CatchEnd)))
+
+                        maybeAddAndApply(finallyp, ScalaTokenConstants.Finally)
+                    })
+                case Term.Apply(fun, args) =>
+                    // `f()` can also be written as `f`, so we simply don't see `f()` as a function call
+                    // But Java style function calls with no args should still be recognized
+                    TR(traverse = _ => {
+                        add(Apply, fun, fromEnd=false)
+                        apply(fun)
+                        for (arg <- args) {
+                            add(Argument, arg, fromEnd = false)
+                            arg match {
+                                case Term.Assign(lhs, rhs) =>
+                                    apply(lhs)
+                                    apply(rhs)
+                                case _ => apply(arg)
+                            }
+                        }
+                    })
+                case Term.NewAnonymous(_) => TR(Some(NewCreationBegin), Some(NewCreationEnd))
+                case Term.Return(_) => TR(Some(ScalaTokenConstants.Return))
+                case Term.Match(expr, cases) => TR(traverse = _ => {
+                    add(MatchBegin, tree, fromEnd=false)
+                    apply(expr)
+                    processCases(cases)
+                    add(MatchEnd, tree, fromEnd=true)
+                })
+                case Term.Throw(_) => TR(Some(Throw))
+                case Term.Function(_) => TR(Some(FunctionBegin), Some(FunctionEnd))
+                case Term.PartialFunction(cases) => TR(traverse = _ => {
+                    add(PartialFunctionBegin, tree, fromEnd=false)
+                    processCases(cases)
+                    add(PartialFunctionEnd, tree, fromEnd=true)
+                })
+                case Term.ForYield(_) => TR(traverse = _ => {
+                    add(ForBodyBegin, tree, fromEnd=false)
+                    add(Yield, tree, fromEnd = false)
+
+                    tree.children.foreach(apply)
+
+                    add(ForBodyEnd, tree, fromEnd = true)
+                })
+                case Term.If(conditionExpr, thenExpression, elseExpression) => TR(traverse = _ => {
+                    add(If, tree, fromEnd=false)
+                    apply(conditionExpr)
+
+                    enclose(thenExpression, TR(Some(IfBegin), Some(IfEnd)))
+
+                    elseExpression match {
+                        case Lit.Unit() => apply(elseExpression)
+                        case _ =>
+                            val elseStart = tree.pos.text.indexOf("else", thenExpression.pos.end - tree.pos.start)
+                            val elsePosition = Position.Range(tree.pos.input, tree.pos.start + elseStart, tree.pos.start + elseStart + 4)
+                            add(Else, elsePosition.startLine + 1, elsePosition.startColumn + 1, elsePosition.text.length)
+                            enclose(elseExpression, TR(Some(ElseBegin), Some(ElseEnd)))
+                    }
+                })
+
+                case scala.meta.Pkg(_) => TR(Some(Package))
+                case scala.meta.Import(_) => TR(Some(ScalaTokenConstants.Import))
+
+                case Defn.Def(mods, name, typeParams, paramss, _, body) =>
+                    TR(traverse = _ => {
+                        applyRecursively(mods)
+                        add(MethodDef, name, fromEnd = false)
+                        for (tParam <- typeParams) add(TypeParameter, tParam, fromEnd = false)
+                        for (params <- paramss)
+                            for (param <- params) add(Parameter, param, fromEnd = false)
+
+                        enclose(body, TR(Some(MethodBegin), Some(MethodEnd)))
+                    })
+                case Defn.Macro(mods, name, tparams, paramss, declType, body) => TR(Some(Macro), traverse = _ => {
+                    applyRecursively(mods)
+                    applyRecursively(tparams)
+                    applyRecursively(paramss)
+                    enclose(body, TR(Some(MacroBegin), Some(MacroEnd)))
+                })
+                case Defn.Class(_) =>
+                    TR(Some(ClassBegin), Some(ClassEnd))
+                case Defn.Object(_) => TR(Some(ObjectBegin), Some(ObjectEnd))
+                case Defn.Trait(_) => TR(Some(TraitBegin), Some(TraitEnd))
+                case Defn.Type(_) => TR(Some(ScalaTokenConstants.Type))
+                case Defn.Var(mods, pats, decltype, rhs) => TR(traverse = _ => {
+                    apply(mods)
+                    for (pat <- pats) {
+                        add(VariableDefinition, pat, fromEnd=false)
+                        apply(pat)
+                        apply(decltype)
+
+                        rhs match {
+                            case Some(realRhs) =>
+                                add(Assign, realRhs, fromEnd=false)
+                                apply(realRhs)
+                            case None =>
+                        }
+                    }
+                })
+                case Defn.Val(mods, pats, decltype, rhs) => TR(traverse = _ => {
+                    apply(mods)
+                    for (pat <- pats) {
+                        add(VariableDefinition, pat, fromEnd=false)
+                        apply(pat)
+                        apply(decltype)
+
+                        add(Assign, rhs, fromEnd=false)
+                        apply(rhs)
+                    }
+
+                })
+
+                case Decl.Var(_) => TR(Some(VariableDefinition))
+                case Decl.Val(_) => TR(Some(VariableDefinition))
+                case Decl.Def(_) => TR(Some(MethodBegin), Some(MethodEnd))
+                case Decl.Type(_) => TR(Some(ScalaTokenConstants.Type))
+
+                case Ctor.Secondary(_) =>
+                    TR(Some(ConstructorBegin), Some(ConstructorEnd))
+                case Init(tpe, name, argss) if argss.nonEmpty => TR(traverse = _ => {
+                    for (args <- argss) {
+                        for (arg <- args) {
+                            add(Argument, arg, fromEnd = false)
+                            apply(arg)
+                        }
+                    }
+                })
+                case Enumerator.Guard(_) => TR(Some(Guard))
+
+                case Term.Param(_) => TR(traverse = _ => add(Parameter, tree, fromEnd = false))
+                case Term.NewAnonymous(_) => TR(Some(ClassBegin), Some(ClassEnd))
+                case Term.ApplyInfix(_, op, _, _) if op.value.contains("=") && !Array("==", "!=").contains(op.value) => TR(Some(Assign))
+                case Term.ApplyInfix(fun, op, typeArgs, args) if isNotArithmetic(op) => TR(traverse = _ => {
+                    add(Apply, tree, fromEnd = false)
+                    apply(fun)
+                    for (typeArg <- typeArgs) {
+                        add(TypeArgument, typeArg, fromEnd = false)
+                        apply(typeArg)
+                    }
+                    for (arg <- args) {
+                        add(Argument, arg, fromEnd = false)
+                        apply(arg)
+                    }
+                })
+                case Term.Select(refObj, member) =>
+                    TR(traverse = _ => {
+                        apply(refObj)
+                        add(ScalaTokenConstants.Member, member, fromEnd = false)
+                        apply(member)
+                    })
+                case Term.ApplyType(_, typeArgs) => TR(traverse = _ => {
+                    add(Apply, tree, fromEnd = false)
+                    for (typeArg <- typeArgs) add(TypeArgument, typeArg, fromEnd = false)
+                })
+                case Term.New(_) => TR(Some(NewObject))
+                case Self(_) => TR(Some(SelfType))
+                case block@Term.Block(_) if block.parent.nonEmpty && block.parent.get.isInstanceOf[Term.Apply] =>
+                    TR(Some(BlockStart), Some(BlockEnd))
+                case Enumerator.Generator(_) => TR(Some(EnumGenerator))
+                case meta.Type.Param(_) => TR(Some(TypeParameter))
+
+                case _ => TR()
             }
+        }
 
-            maybeAddAndApply(finallyExpr, ScalaTokenConstants.Finally)
-          })
-        case Term.TryWithHandler(expr, catchp, finallyp) => TR(Some(TryBegin),
-          traverse = _ => {
-            apply(expr)
-            add(CatchBegin, catchp, fromEnd=false)
-            apply(catchp)
-            add(CatchEnd, catchp, fromEnd = true)
-
-            maybeAddAndApply(finallyp, ScalaTokenConstants.Finally)
-          })
-        case Term.Apply(fun, args) =>
-          // `f()` can also be written as `f`, so we simply don't see `f()` as a function call
-          // But Java style function calls with no args should still be recognized
-          TR(traverse = _ => {
-            add(Apply, tree, fromEnd=false)
-            apply(fun)
-            for (arg <- args) {
-              add(Argument, arg, fromEnd = false)
-              arg match {
-                case Term.Assign(lhs, rhs) =>
-                  apply(lhs)
-                  apply(rhs)
-                case _ => apply(arg)
-              }
-            }
-          })
-        case Term.NewAnonymous(_) => TR(Some(NewCreationBegin), Some(ScalaTokenConstants.NewCreationBegin))
-        case Term.Return(_) => TR(Some(ScalaTokenConstants.Return))
-        case Term.Match(expr, cases) => TR(traverse = _ => {
-          add(MatchBegin, tree, fromEnd=false)
-          apply(expr)
-          processCases(cases)
-          add(MatchEnd, tree, fromEnd=true)
+        def applyRecursively[T](els: List[T]): Unit = els.foreach( {
+             case tree: Tree => apply(tree)
+             case treeList: List[_] => applyRecursively(treeList)
         })
-        case Term.Throw(_) => TR(Some(Throw))
-        case Term.Function(_) => TR(Some(FunctionBegin), Some(FunctionEnd))
-        case Term.PartialFunction(cases) => TR(traverse = _ => {
-          add(PartialFunctionBegin, tree, fromEnd=false)
-          processCases(cases)
-          add(PartialFunctionEnd, tree, fromEnd=true)
-        })
-        case Term.ForYield(_) => TR(traverse = _ => {
-          add(ForBegin, tree, fromEnd=false)
-          add(Yield, tree, fromEnd = false)
-
-          tree.children.foreach(apply)
 
-          add(ForEnd, tree, fromEnd = true)
-        })
-        case Term.If(conditionExpr, thenExpression, elseExpression) => TR(traverse = _ => {
-          add(If, tree, fromEnd=false)
-          apply(conditionExpr)
-          add(IfBegin, thenExpression, fromEnd=false)
+        override def apply(tree: Tree): Unit = {
+            val record : TR = doMatch(tree)
 
-          apply(thenExpression)
-          add(IfEnd, thenExpression, fromEnd=true)
+            record.before match {
+                case Some(value) => add(value, tree, fromEnd = false)
+                case None =>
+            }
 
-          elseExpression match {
-            case Lit.Unit() => apply(elseExpression)
-            case _ =>
-              val elseStart = tree.pos.text.indexOf("else", thenExpression.pos.end - tree.pos.start)
-              val elsePosition = Position.Range(tree.pos.input, tree.pos.start + elseStart, tree.pos.start + elseStart + 4)
-              add(Else, elsePosition.startLine + 1, elsePosition.startColumn + 1, elsePosition.text.length)
-              add(ElseBegin, elseExpression, fromEnd = false)
-              apply(elseExpression)
-              add(ElseEnd, elseExpression, fromEnd = true)
-          }
-        })
+            record.traverse(tree)
 
-        case scala.meta.Pkg(_) => TR(Some(Package))
-        case scala.meta.Import(_) => TR(Some(ScalaTokenConstants.Import))
-
-        case Defn.Def(_) => TR(Some(MethodBegin), Some(MethodEnd))
-        case Defn.Macro(_) => TR(Some(MacroBegin), Some(MacroEnd))
-        case Defn.Class(_) => TR(Some(ClassBegin), Some(ClassEnd))
-        case Defn.Object(_) => TR(Some(ObjectBegin), Some(ObjectEnd))
-        case Defn.Trait(_) => TR(Some(TraitBegin), Some(TraitEnd))
-        case Defn.Type(_) => TR(Some(ScalaTokenConstants.Type))
-        case Defn.Var(mods, pats, decltype, rhs) => TR(traverse = _ => {
-          apply(mods)
-          for (pat <- pats) {
-            add(VariableDefinition, pat, fromEnd=false)
-            apply(pat)
-            apply(decltype)
-
-            rhs match {
-              case Some(realRhs) =>
-                add(Assign, realRhs, fromEnd=false)
-                apply(realRhs)
-              case None =>
+            record.after match {
+                case Some(value) => add(value, tree, fromEnd = true)
+                case None =>
             }
-          }
-        })
-        case Defn.Val(mods, pats, decltype, rhs) => TR(traverse = _ => {
-          apply(mods)
-          for (pat <- pats) {
-            add(VariableDefinition, pat, fromEnd=false)
-            apply(pat)
-            apply(decltype)
+        }
 
-            add(Assign, rhs, fromEnd=false)
-            apply(rhs)
-          }
+        def enclose(tree: Tree, record: TR): Unit = {
+            record.before match {
+                case Some(value) => add(value, tree, fromEnd = false)
+                case None =>
+            }
 
-        })
+            apply(tree)
 
-        case Decl.Var(_) => TR(Some(VariableDefinition))
-        case Decl.Var(_) => TR(Some(VariableDefinition))
-        case Decl.Def(_) => TR(Some(MethodBegin), Some(MethodEnd))
-        case Decl.Type(_) => TR(Some(ScalaTokenConstants.Type))
-
-        case Ctor.Secondary(_) => TR(Some(ConstructorBegin), Some(ConstructorEnd))
-
-        case Enumerator.Guard(_) => TR(Some(Guard))
-
-        case Term.Param(_) => TR(traverse = _ => add(Parameter, tree, fromEnd = false))
-        case Term.NewAnonymous(_) => TR(Some(ClassBegin), Some(ClassEnd))
-        case Term.ApplyInfix(_, op, _, _) if op.value.contains("=") && !Array("==", "!=").contains(op.value) => TR(Some(Assign))
-        case Term.ApplyInfix(fun, op, typeArgs, args) if isNotArithmetic(op) => TR(traverse = _ => {
-          add(Apply, tree, fromEnd = false)
-          apply(fun)
-          for (typeArg <- typeArgs) {
-            add(TypeArgument, typeArg, fromEnd = false)
-            apply(typeArg)
-          }
-          for (arg <- args) {
-            add(Argument, arg, fromEnd = false)
-            apply(arg)
-          }
-        })
-        case Term.ApplyType(fun, typeArgs) => TR(traverse = _ => {
-          add(Apply, tree, fromEnd = false)
-          for (typeArg <- typeArgs) add(TypeArgument, typeArg, fromEnd = false)
-        })
-        case Term.New(_) => TR(Some(NewObject))
-        case Self(_) => TR(Some(SelfType))
-        case Term.Block(_) => TR(Some(BlockStart), Some(BlockEnd))
-        case Enumerator.Generator(_) => TR(Some(EnumGenerator))
-        case _ => TR()
-      }
+            record.after match {
+                case Some(value) => add(value, tree, fromEnd = true)
+                case None =>
+            }
+        }
     }
 
-    override def apply(tree: Tree): Unit = {
-      val res = doMatch(tree)
 
-      res.before match {
-        case Some(value) => add(value, tree, fromEnd = false)
-        case None =>
-      }
 
-      res.traverse(tree)
+    def parse(dir: File, files: Array[String]): TokenList = {
+        tokens = new TokenList
+        errors = 0
 
-      res.after match {
-        case Some(value) => add(value, tree, fromEnd = true)
-        case None =>
-      }
-    }
-  }
-
-
-
-  def parse(dir: File, files: Array[String]): TokenList = {
-    tokens = new TokenList
-    errors = 0
+        for (file <- files) {
+            currentFile = file
+            // getProgram.print(null, "Parsing file" + file + "\n")
+            if (!parseFile(dir, file)) {
+                errors += 1
+            }
+            System.gc()
+        }
 
-    for (file <- files) {
-      currentFile = file
-      // getProgram.print(null, "Parsing file" + file + "\n")
-      if (!parseFile(dir, file)) {
-        errors += 1
-      }
-      System.gc()
+        tokens
     }
 
-    tokens
-  }
-
-  def parseFile(dir: File, file: String): Boolean = {
-    currentFile = file
-
-    try {
-      val f = new File(dir, file)
-      val bytes = java.nio.file.Files.readAllBytes(f.toPath)
-      val text = new String(bytes, "UTF-8")
-      val input = Input.VirtualFile(f.getPath, text)
-      val ast = input.parse[Source].get
-      traverser(ast)
-
-      add(FileEnd, text.count(_ == '\n')-1, 0, 0)
-    } catch {
-      case e: Throwable =>
-        e.printStackTrace()
-        return false
+    def parseFile(dir: File, file: String): Boolean = {
+        currentFile = file
+
+        try {
+            val f = new File(dir, file)
+            val bytes = java.nio.file.Files.readAllBytes(f.toPath)
+            val text = new String(bytes, "UTF-8")
+            val input = Input.VirtualFile(f.getPath, text)
+            val ast = input.parse[Source].get
+            traverser(ast)
+
+            add(FileEnd, text.count(_ == '\n')-1, 0, 0)
+        } catch {
+            case e: Throwable =>
+                e.printStackTrace()
+                return false
+        }
+
+        true
     }
 
-    true
-  }
-
-  private def add(tType: ScalaTokenConstants.Value, line: Int, column: Int, length: Int): Unit = {
-    tokens.addToken(new ScalaToken(tType.id, currentFile, line, column, length))
-  }
+    private def add(tType: ScalaTokenConstants.Value, line: Int, column: Int, length: Int): Unit = {
+        tokens.addToken(new ScalaToken(tType.id, currentFile, line, column, length))
+    }
 
 
 
-  private def add(tType: ScalaTokenConstants.Value, node: Tree, fromEnd: Boolean): Unit = {
-    if (node.pos.text.nonEmpty) {
-      // SELF type tokens with no text content mess up the sequence
-      tokens.addToken(new ScalaToken(tType, currentFile, node.pos, fromEnd))
+    private def add(tType: ScalaTokenConstants.Value, node: Tree, fromEnd: Boolean): Unit = {
+        if (node.pos.text.nonEmpty) {
+            // SELF type tokens with no text content mess up the sequence
+            tokens.addToken(new ScalaToken(tType, currentFile, node.pos, fromEnd))
+        }
     }
-  }
 
 }

jplag.frontend.scala/src/test/resources/de/jplag/scala/Typers.scala

@@ -1,79 +0,0 @@
-package mypackage
-
-import scala.otherpackage
-
-trait MyTrait extends SuperTrait with OtherTrait {
-  self: SelfType =>
-
-  final val valAssignee = assigned
-
-  private val rightAssocValDefs = new mutable.AnyRefMap[Symbol, Tree]
-
-  sealed abstract class MyClass[+T] {
-    def attribute: Boolean
-
-    @inline final def methodWithTypeParam[U](none: => U)(f: T => U): U = this match {
-      case SubType(attr) => f(attr)
-      case _                        => none
-    }
-
-  }
-
-  object MyObject extends SuperType {
-
-    override def apply(t: Type): Type = {
-      def checkNoEscape(sym: Symbol): Unit = {
-        while (o != NoSymbol && o != sym.owner && o != sym.owner.linkedClassOfClass &&
-          !o.isLocalToBlock && !o.isPrivate &&
-          !o.privateWithin.hasTransOwner(sym.owner))
-          o = o.owner
-      }
-
-    }
-  }
-
-  private final val typerFreshNameCreators = perRunCaches.newAnyRefMap[Symbol, FreshNameCreator]()
-
-  abstract class MyClass2(constrArg: ParamType) extends SuperClass with Trait1 with Trait2 {
-
-    val anonymousSubclassObject = new scala.AnyVal {   }
-
-    def applyImplicitArgs(fun: Tree): Tree = fun.tpe match {
-      case MethodType(params, _) =>
-        var mkArg: (Name, Tree) => Tree = (_, tree) => tree
-
-        for(param <- params) {
-          var paramTp = param.tpe
-        }
-
-      case x => throw new MatchError(x)
-    }
-
-    def tryCatchFinally() = try {
-      if (true) false
-      else otherMethod(arg)
-    } catch {
-      case ex: CyclicReference => throw ex
-      case _ => return true
-    } finally {
-      cleanUp()
-    }
-
-    def yieldingMethodWithGuard(values: List[Int]): List[Int] =
-      for (value <- values if value > 0) yield {
-        value
-      }
-
-    def validateParentClasses(parents: List[Tree], selfType: Type, clazzIsTrait: Boolean): Unit = {
-      (parent.tpe :: ps).collectFirst {
-        case p if hasTraitParams(p.typeSymbol) =>
-          p.typeSymbol.attachments.get[DottyParameterisedTrait].foreach(attach =>
-            pending += ParentIsScala3TraitError(parent, p.typeSymbol, attach.params, psym)
-          )
-      }
-    }
-
-
-  }
-
-}