Set up Scala frontend initially

jplag.frontend.scala/pom.xml

@@ -0,0 +1,157 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>de.jplag</groupId>
+    <artifactId>aggregator</artifactId>
+    <version>${revision}</version>
+  </parent>
+  <artifactId>scala</artifactId>
+
+  <properties>
+    <encoding>UTF-8</encoding>
+    <scala.version>2.12.6</scala.version>
+    <scala.compat.version>2.12</scala.compat.version>
+    <spec2.version>4.2.0</spec2.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.scala-lang</groupId>
+      <artifactId>scala-library</artifactId>
+      <version>${scala.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>net.alchim31.maven</groupId>
+      <artifactId>scala-maven-plugin</artifactId>
+      <version>4.6.2</version>
+    </dependency>
+    <dependency>
+      <groupId>de.jplag</groupId>
+      <artifactId>frontend-utils</artifactId>
+    </dependency>
+
+    <!-- https://mvnrepository.com/artifact/org.scalameta/scalameta -->
+    <dependency>
+      <groupId>org.scalameta</groupId>
+      <artifactId>scalameta_2.12</artifactId>
+      <version>4.5.9</version>
+    </dependency>
+
+
+    <!-- Test -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.13.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.scalatest</groupId>
+      <artifactId>scalatest_${scala.compat.version}</artifactId>
+      <version>3.2.12</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.specs2</groupId>
+      <artifactId>specs2-core_${scala.compat.version}</artifactId>
+      <version>${spec2.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.specs2</groupId>
+      <artifactId>specs2-junit_${scala.compat.version}</artifactId>
+      <version>${spec2.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>de.jplag</groupId>
+      <artifactId>frontend-testutils</artifactId>
+      <version>${revision}</version>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <sourceDirectory>src/main/scala</sourceDirectory>
+    <testSourceDirectory>src/test/java</testSourceDirectory>
+    <plugins>
+      <plugin>
+        <!-- see http://davidb.github.com/scala-maven-plugin -->
+        <groupId>net.alchim31.maven</groupId>
+        <artifactId>scala-maven-plugin</artifactId>
+        <version>3.3.2</version>
+        <executions>
+          <execution>
+            <goals>
+              <goal>compile</goal>
+              <goal>testCompile</goal>
+            </goals>
+            <configuration>
+              <args>
+                <arg>-dependencyfile</arg>
+                <arg>${project.build.directory}/.scala_dependencies</arg>
+              </args>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>2.21.0</version>
+        <configuration>
+          <!-- Tests will be run with scalatest-maven-plugin instead -->
+          <skipTests>true</skipTests>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.scalatest</groupId>
+        <artifactId>scalatest-maven-plugin</artifactId>
+        <version>2.0.0</version>
+        <configuration>
+          <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
+          <junitxml>.</junitxml>
+          <filereports>TestSuiteReport.txt</filereports>
+          <!-- Comma separated list of JUnit test class names to execute -->
+          <jUnitClasses>samples.AppTest</jUnitClasses>
+        </configuration>
+        <executions>
+          <execution>
+            <id>test</id>
+            <goals>
+              <goal>test</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <version>2.4</version>
+        <configuration>
+          <descriptorRefs>
+            <descriptorRef>jar-with-dependencies</descriptorRef>
+          </descriptorRefs>
+          <archive>
+            <manifest>
+              <mainClass>de.jplag.scala.Parser</mainClass>
+              <packageName>jplag.scala</packageName>
+            </manifest>
+          </archive>
+        </configuration>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

jplag.frontend.scala/src/main/scala/de/jplag/scala/Language.scala

@@ -0,0 +1,32 @@
+package de.jplag.scala
+
+import de.jplag.{ErrorConsumer, TokenList}
+
+import java.io.File
+
+
+class Language(val consumer: ErrorConsumer) extends de.jplag.Language {
+  private val parser = new Parser(consumer)
+
+  override def suffixes: Array[String] = {
+    Array(".scala")
+  }
+
+  override def getName = "Scala parser"
+
+  override def getShortName = "scala"
+
+  override def minimumTokenMatch = 8
+
+  override def parse(dir: File, files: Array[String]): TokenList = this.parser.parse(dir, files)
+
+  override def hasErrors: Boolean = this.parser.hasErrors
+
+  override def supportsColumns = true
+
+  override def isPreformatted = true
+
+  override def usesIndex = false
+
+  override def numberOfTokens: Int = ScalaTokenConstants.maxId
+}

jplag.frontend.scala/src/main/scala/de/jplag/scala/Parser.scala

@@ -0,0 +1,240 @@
+package de.jplag.scala
+
+import de.jplag.{AbstractParser, ErrorConsumer, TokenList}
+
+import java.io.File
+import ScalaTokenConstants._
+
+import scala.meta.Term.ApplyInfix
+import scala.meta._
+
+
+class Parser(consumer: ErrorConsumer) extends AbstractParser(consumer) {
+  private var currentFile : String = _
+
+  var tokens : TokenList = _
+
+  private val traverser: Traverser = new Traverser {
+
+    case class TR(before: Option[ScalaTokenConstants.Value] = None,
+                  after: Option[ScalaTokenConstants.Value] = None,
+                  traverse: Tree => Unit = _.children.foreach(traverser.apply))
+
+    private def maybeAddAndApply(tree: Option[Tree], token: ScalaTokenConstants.Value): Unit = tree match {
+      case Some(exp) =>
+        add(token, exp, fromEnd = false)
+        apply(exp)
+      case None =>
+    }
+
+    private def processCases(cases: List[Case]) : Unit = cases.foreach {
+      c =>
+        add(ScalaTokenConstants.Case, c, fromEnd=false)
+        apply(c)
+    }
+
+    private def doMatch(tree: Tree): TR = {
+      tree match {
+        case Term.Do(_) => TR(Some(DoBegin), Some(DoEnd))
+        case Term.Assign(_) => TR(Some(Assign), None)
+        case Term.While(_) => TR(Some(WhileBegin), Some(WhileEnd))
+        case Term.For(_) => TR(Some(ForBegin), Some(ForEnd))
+        case Term.Try(expr, catchp, finallyp) => TR(Some(TryBegin),
+          traverse = _ => {
+            apply(expr)
+            if (catchp.nonEmpty) {
+              val start = catchp.head.pos
+              val end = catchp.last.pos
+              val len = end.endLine - start.startLine
+
+              add(CatchBegin, start.startLine, start.startColumn, len)
+              processCases(catchp)
+              add(CatchEnd, end.endLine, end.endColumn, len)
+            }
+
+            maybeAddAndApply(finallyp, ScalaTokenConstants.Finally)
+          })
+        case Term.TryWithHandler(expr, catchp, finallyp) => TR(Some(TryBegin),
+          traverse = _ => {
+            apply(expr)
+            add(CatchBegin, catchp, fromEnd=false)
+            apply(catchp)
+            add(CatchEnd, catchp, fromEnd = true)
+
+            maybeAddAndApply(finallyp, ScalaTokenConstants.Finally)
+          })
+        case Term.Apply(fun, args) =>
+          // `f()` can also be written as `f`, so we simply don't see `f()` as a function call
+          // But Java style function calls with no args should still be recognized
+          TR(traverse = _ => {
+            add(Apply, tree, fromEnd=false)
+            apply(fun)
+            for (arg <- args) {
+              add(Argument, arg, fromEnd = false)
+              arg match {
+                case Term.Assign(lhs, rhs) =>
+                  apply(lhs)
+                  apply(rhs)
+                case _ => apply(arg)
+              }
+            }
+          })
+        case Term.NewAnonymous(_) => TR(Some(NewCreationBegin), Some(ScalaTokenConstants.NewCreationBegin))
+        case Term.Return(_) => TR(Some(ScalaTokenConstants.Return))
+        case Term.Match(expr, cases) => TR(traverse = _ => {
+          add(MatchBegin, tree, fromEnd=false)
+          apply(expr)
+          processCases(cases)
+          add(MatchEnd, tree, fromEnd=true)
+        })
+        case Term.Throw(_) => TR(Some(Throw))
+        case Term.Function(_) => TR(Some(FunctionBegin), Some(FunctionEnd))
+        case Term.PartialFunction(cases) => TR(traverse = _ => {
+          add(PartialFunctionBegin, tree, fromEnd=false)
+          processCases(cases)
+          add(PartialFunctionEnd, tree, fromEnd=true)
+        })
+        case Term.ForYield(_) => TR(traverse = _ => {
+          add(ForBegin, tree, fromEnd=false)
+          add(Yield, tree, fromEnd = false)
+
+          tree.children.foreach(apply)
+
+          add(ForEnd, tree, fromEnd = true)
+        })
+        case Term.If(cond, thenp, elsep) => TR(traverse = _ => {
+          add(IfBegin, tree, fromEnd=false)
+          apply(cond)
+
+          apply(thenp)
+
+          elsep match {
+            case Lit.Unit() =>
+            case _ =>
+              add(ScalaTokenConstants.Else, elsep, fromEnd=false)
+          }
+          apply(elsep)
+
+          add(IfEnd, tree, fromEnd=true)
+        })
+
+        case scala.meta.Pkg(_) => TR(Some(Package))
+        case scala.meta.Import(_) => TR(Some(ScalaTokenConstants.Import))
+
+        case Defn.Def(_) => TR(Some(MethodBegin), Some(MethodEnd))
+        case Defn.Macro(_) => TR(Some(MacroBegin), Some(MacroEnd))
+        case Defn.Class(_) => TR(Some(ClassBegin), Some(ClassEnd))
+        case Defn.Object(_) => TR(Some(ObjectBegin), Some(ObjectEnd))
+        case Defn.Trait(_) => TR(Some(TraitBegin), Some(TraitEnd))
+        case Defn.Type(_) => TR(Some(ScalaTokenConstants.Type))
+        case Defn.Var(mods, pats, decltype, rhs) => TR(traverse = _ => {
+          apply(mods)
+          for (pat <- pats) {
+            add(VariableDefinition, pat, fromEnd=false)
+            apply(pat)
+            apply(decltype)
+
+            rhs match {
+              case Some(realRhs) =>
+                add(Assign, realRhs, fromEnd=false)
+                apply(realRhs)
+              case None =>
+            }
+          }
+        })
+        case Defn.Val(mods, pats, decltype, rhs) => TR(traverse = _ => {
+          apply(mods)
+          for (pat <- pats) {
+            add(VariableDefinition, pat, fromEnd=false)
+            apply(pat)
+            apply(decltype)
+
+            add(Assign, rhs, fromEnd=false)
+            apply(rhs)
+          }
+
+        })
+
+        case Decl.Var(_) => TR(Some(VariableDefinition))
+        case Decl.Var(_) => TR(Some(VariableDefinition))
+        case Decl.Def(_) => TR(Some(MethodBegin), Some(MethodEnd))
+        case Decl.Type(_) => TR(Some(ScalaTokenConstants.Type))
+
+        case Ctor.Secondary(_) => TR(Some(ConstructorBegin), Some(ConstructorEnd))
+
+        case Enumerator.Guard(_) => TR(Some(Guard))
+
+        case Term.Param(_) => TR(traverse = _ => add(Parameter, tree, fromEnd = false))
+        case Term.NewAnonymous(_) => TR(Some(ClassBegin), Some(ClassEnd))
+        case Term.ApplyInfix(_, op, _, _) if op.value.contains("=") && !op.value.equals("==") =>
+          TR(Some(Assign), None)
+
+        case _ => TR()
+      }
+    }
+
+    override def apply(tree: Tree): Unit = {
+      val res = doMatch(tree)
+
+      res.before match {
+        case Some(value) => add(value, tree, fromEnd = false)
+        case None =>
+      }
+
+      res.traverse(tree)
+
+      res.after match {
+        case Some(value) => add(value, tree, fromEnd = true)
+        case None =>
+      }
+    }
+  }
+
+
+
+  def parse(dir: File, files: Array[String]): TokenList = {
+    tokens = new TokenList
+    errors = 0
+
+    for (file <- files) {
+      currentFile = file
+      // getProgram.print(null, "Parsing file" + file + "\n")
+      if (!parseFile(dir, file)) {
+        errors += 1
+      }
+      System.gc()
+    }
+
+    tokens
+  }
+
+  def parseFile(dir: File, file: String): Boolean = {
+    currentFile = file
+
+    try {
+      val f = new File(dir, file)
+      val bytes = java.nio.file.Files.readAllBytes(f.toPath)
+      val text = new String(bytes, "UTF-8")
+      val input = Input.VirtualFile(f.getPath, text)
+      val ast = input.parse[Source].get
+      traverser(ast)
+
+      add(FileEnd, text.count(_ == '\n') + 1, 0, 0)
+    } catch {
+      case e: Throwable =>
+        e.printStackTrace()
+        return false
+    }
+
+    true
+  }
+
+  private def add(typ: ScalaTokenConstants.Value, line: Int, column: Int, length: Int): Unit = {
+    tokens.addToken(new ScalaToken(typ.id, currentFile, line, column, length))
+  }
+
+
+  private def add(tType: ScalaTokenConstants.Value, node: Tree, fromEnd: Boolean): Unit = {
+    tokens.addToken(new ScalaToken(node, fromEnd, currentFile, tType))
+  }
+}

jplag.frontend.scala/src/main/scala/de/jplag/scala/ScalaTokenConstants.scala

@@ -0,0 +1,116 @@
+package de.jplag.scala
+
+import de.jplag.{Token, TokenConstants}
+
+object ScalaTokenConstants extends Enumeration with TokenConstants {
+
+  val FileEnd: Value = Value(TokenConstants.FILE_END, "<EOF>")
+  val SeparatorToken: Value = Value(TokenConstants.SEPARATOR_TOKEN, "--------")
+  val Package: Value = Value("PACKAGE")
+  val Import: Value = Value("IMPORT")
+  val ClassBegin: Value = Value("CLASS{")
+  val ClassEnd: Value = Value("}CLASS")
+  val MethodBegin: Value = Value("METHOD{")
+  val MethodEnd: Value = Value("}METHOD")
+  val VariableDefinition: Value = Value("VAR_DEF")
+  val DoBegin: Value = Value("DO{")
+  val DoEnd: Value = Value("}DO")
+  val WhileBegin: Value = Value("WHILE{")
+  val WhileEnd: Value = Value("}WHILE")
+  val ForBegin: Value = Value("FOR{")
+  val ForEnd: Value = Value("}FOR")
+  val Case: Value = Value("CASE")
+  val TryBegin: Value = Value("TRY{")
+  val CatchBegin: Value = Value("CATCH{")
+  val CatchEnd: Value = Value("}CATCH")
+  val Finally: Value = Value("FINALLY")
+  val IfBegin: Value = Value("IF{")
+  val Else: Value = Value("ELSE{")
+  val IfEnd: Value = Value("}IF")
+  val Return: Value = Value("RETURN")
+  val Throw: Value = Value("THROW")
+  val NewCreationBegin: Value = Value("NEW{")
+  val NewCreationEnd: Value = Value("}NEW")
+  val Apply: Value = Value("APPLY")
+  val Assign: Value = Value("ASSIGN")
+  val TraitBegin: Value = Value("TRAIT{")
+  val TraitEnd: Value = Value("}TRAIT")
+  val ConstructorBegin: Value = Value("CONSTR{")
+  val ConstructorEnd: Value = Value("}CONSTR")
+  val InitBegin: Value = Value("INIT{")
+  val InitEnd: Value = Value("}INIT")
+  val ArrayInitBegin: Value = Value("ARRAY(){")
+  val ArrayInitEnd: Value = Value("}ARRAY")
+  val MatchBegin: Value = Value("MATCH{")
+  val MatchEnd: Value = Value("}MATCH")
+  val Guard: Value = Value("GUARD")
+  val Anno: Value = Value("ANNO")
+  val AnnoMarker: Value = Value("ANNO_MARKER")
+  val AnnoMBegin: Value = Value("ANNO_M{")
+  val AnnoMEnd: Value = Value("}ANNO_M")
+  val AnnoTBegin: Value = Value("ANNO_T{")
+  val AnnoTEnd: Value = Value("}ANNO_T")
+  val AnnoCBegin: Value = Value("ANNO_C{")
+  val AnnoCEnd: Value = Value("}ANNO_C")
+  val ObjectBegin: Value = Value("OBJECT{")
+  val ObjectEnd: Value = Value("}OBJECT")
+  val MacroBegin: Value = Value("MACRO{")
+  val MacroEnd: Value = Value("}MACRO")
+  val Type: Value = Value("TYPE")
+
+  val FunctionBegin: Value = Value("FUNC{")
+  val FunctionEnd: Value = Value("}FUNC")
+  val PartialFunctionBegin: Value = Value("PFUNC{")
+  val PartialFunctionEnd: Value = Value("}PFUNC")
+
+  val Yield: Value = Value("YIELD")
+
+  val Parameter: Value = Value("PARAM")
+  val Assignment: Value = Value("ASSIGN")
+  val Argument: Value = Value("ARG")
+}
+
+class ScalaToken(tType: Int,
+                 file: java.lang.String,
+                 var line: Int,
+                 var column: Int,
+                 var length: Int)
+  extends Token(tType, file, line, column, length) {
+
+  def this(node: scala.meta.Tree, fromEnd: Boolean, file: String, typ: ScalaTokenConstants.Value) {
+    this(
+      typ.id,
+      file,
+      (if (fromEnd) node.pos.endLine else node.pos.startLine) + 1,
+      (if (fromEnd) node.pos.endColumn else node.pos.startColumn) + 1,
+      node.toString().length
+    )
+  }
+
+  override def toString: String = {
+    try {
+      ScalaTokenConstants(`type`).toString
+    } catch {
+      case _: Throwable => "Unknown!"
+    }
+  }
+
+  override def setLine(line: Int): Unit = this.line = line
+
+  override def setColumn(column: Int): Unit = this.column = column
+
+  override def setLength(length: Int): Unit = this.length = length
+
+  override def getLine: Int = this.line
+
+  override def getColumn: Int = this.column
+
+  override def getLength: Int = this.length
+
+  def numberOfTokens: Int = ScalaTokenConstants.maxId
+
+  /**
+   * @return a string representation depending on the type of the token.
+   */
+  override protected def type2string(): String = ScalaTokenConstants.apply(tType).toString
+}

jplag.frontend.scala/src/test/java/de/jplag/scala/ScalaFrontendTest.java

@@ -0,0 +1,172 @@
+package de.jplag.scala;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.StreamSupport;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import de.jplag.Token;
+import de.jplag.TokenConstants;
+import de.jplag.TokenList;
+import de.jplag.TokenPrinter;
+import de.jplag.testutils.TestErrorConsumer;
+
+class ScalaFrontendTest {
+
+    /**
+     * Test source file that is supposed to produce a complete set of tokens, i.e. all types of tokens.
+     */
+    private static final String COMPLETE_TEST_FILE = "Complete.scala";
+
+    /**
+     * Regular expression that describes lines consisting only of whitespace and optionally a line comment.
+     */
+    private static final String EMPTY_OR_SINGLE_LINE_COMMENT = "\\s*(//.*|/\\*.*\\*/)?";
+
+    /**
+     * Regular expression that describes lines containing the start of a multiline comment and no code before it.
+     */
+    private static final String DELIMITED_COMMENT_START = "\\s*/\\*(?:(?!\\*/).)*$";
+
+    /**
+     * Regular expression that describes lines containing the end of a multiline comment and no more code after that.
+     */
+    private static final String DELIMITED_COMMENT_END = ".*\\*/\\s*$";
+    private static final String NOT_SET_STRING = "";
+    private static final int NOT_SET = -1;
+
+    private final Logger logger = LoggerFactory.getLogger("Scala frontend test");
+    private final String[] testFiles = new String[] {"Parser.scala"};
+    private final File testFileLocation = Path.of("src", "test", "resources", "de", "jplag", "scala").toFile();
+    private Language language;
+
+    @BeforeEach
+    void setup() {
+        TestErrorConsumer consumer = new TestErrorConsumer();
+        language = new Language(consumer);
+    }
+
+    @Test
+    void parseTestFiles() {
+        for (String fileName : testFiles) {
+            TokenList tokens = language.parse(testFileLocation, new String[] {fileName});
+            String output = TokenPrinter.printTokens(tokens, testFileLocation, List.of(fileName));
+            logger.info(output);
+
+            testSourceCoverage(fileName, tokens);
+            if (fileName.equals(COMPLETE_TEST_FILE)) {
+                testTokenCoverage(tokens, fileName);
+            }
+
+        }
+    }
+
+    /**
+     * Confirms that every type of ScalaToken has a Sting representation associated to it.
+     */
+    @Test
+    void testTokenToString() {
+        var missingTokens = IntStream.range(0, language.numberOfTokens())
+                .mapToObj(type -> new ScalaToken(type, NOT_SET_STRING, NOT_SET, NOT_SET, NOT_SET))
+                .filter(token -> token.type2string().contains("UNKNOWN")).toList();
+
+        if (!missingTokens.isEmpty()) {
+            var typeList = missingTokens.stream().map(Token::getType).map(Object::toString).collect(Collectors.joining(", "));
+            fail("Found token types with no string representation: %s".formatted(typeList));
+        }
+
+    }
+
+    /**
+     * Confirms that the code is covered to a basic extent, i.e. each line of code contains at least one token.
+     * @param fileName a code sample file name
+     * @param tokens the TokenList generated from the sample
+     */
+    private void testSourceCoverage(String fileName, TokenList tokens) {
+        File testFile = new File(testFileLocation, fileName);
+
+        try {
+            List<String> lines = Files.readAllLines(testFile.toPath());
+
+            // All lines that contain code
+            var codeLines = getCodeLines(lines);
+            // All lines that contain token
+            var tokenLines = IntStream.range(0, tokens.size()).mapToObj(tokens::getToken).mapToInt(Token::getLine).distinct().toArray();
+
+            if (codeLines.length > tokenLines.length) {
+                var diffLine = IntStream.range(0, codeLines.length)
+                        .dropWhile(lineIdx -> lineIdx < tokenLines.length && codeLines[lineIdx] == tokenLines[lineIdx]).findFirst();
+                diffLine.ifPresent(
+                        lineIdx -> fail("Line %d of file '%s' is not represented in the token list.".formatted(codeLines[lineIdx], fileName)));
+            }
+            assertArrayEquals(codeLines, tokenLines);
+        } catch (IOException exception) {
+            logger.info("Error while reading test file %s".formatted(fileName), exception);
+            fail();
+        }
+    }
+
+    /**
+     * Gets the line numbers of lines containing actual code, omitting empty lines and comment lines.
+     * @param lines lines of a code file
+     * @return an array of the line numbers of code lines
+     */
+    private int[] getCodeLines(List<String> lines) {
+        // This boxed boolean can be accessed from within the lambda method below
+        var state = new Object() {
+            boolean insideComment = false;
+        };
+
+        var codeLines = IntStream.rangeClosed(1, lines.size()).sequential().filter(idx -> {
+            String line = lines.get(idx - 1);
+            if (line.matches(EMPTY_OR_SINGLE_LINE_COMMENT)) {
+                return false;
+            } else if (line.matches(DELIMITED_COMMENT_START)) {
+                state.insideComment = true;
+                return false;
+            } else if (state.insideComment) {
+                // This fails if code follows after '*/'. If the code is formatted well, this should not happen.
+                if (line.matches(DELIMITED_COMMENT_END)) {
+                    state.insideComment = false;
+                }
+                return false;
+            }
+            return true;
+        });
+
+        return codeLines.toArray();
+
+    }
+
+    /**
+     * Confirms that all Token types are 'reachable' with a complete code example.
+     * @param tokens TokenList which is supposed to contain all types of tokens
+     * @param fileName The file name of the complete code example
+     */
+    private void testTokenCoverage(TokenList tokens, String fileName) {
+        var foundTokens = StreamSupport.stream(tokens.allTokens().spliterator(), true).mapToInt(Token::getType).sorted().distinct().toArray();
+        // Exclude SEPARATOR_TOKEN, as it does not occur
+        var allTokens = IntStream.range(0, language.numberOfTokens()).filter(i -> i != TokenConstants.SEPARATOR_TOKEN).toArray();
+
+        if (allTokens.length > foundTokens.length) {
+            var diffLine = IntStream.range(0, allTokens.length)
+                    .dropWhile(lineIdx -> lineIdx < foundTokens.length && allTokens[lineIdx] == foundTokens[lineIdx]).findFirst();
+            diffLine.ifPresent(lineIdx -> fail("Token type %s was not found in the complete code example '%s'."
+                    .formatted(new ScalaToken(allTokens[lineIdx], fileName, -1, -1, -1).type2string(), fileName)));
+        }
+        assertArrayEquals(allTokens, foundTokens);
+    }
+
+}

jplag.frontend.scala/src/test/resources/de/jplag/scala/Parser.scala

@@ -0,0 +1,240 @@
+package de.jplag.scala
+
+import de.jplag.{AbstractParser, ErrorConsumer, TokenList}
+
+import java.io.File
+import ScalaTokenConstants._
+
+import scala.meta.Term.ApplyInfix
+import scala.meta._
+
+
+class Parser(consumer: ErrorConsumer) extends AbstractParser(consumer) {
+  private var currentFile : String = _
+
+  var tokens : TokenList = _
+
+  private val traverser: Traverser = new Traverser {
+
+    case class TR(before: Option[ScalaTokenConstants.Value] = None,
+                  after: Option[ScalaTokenConstants.Value] = None,
+                  traverse: Tree => Unit = _.children.foreach(traverser.apply))
+
+    private def maybeAddAndApply(tree: Option[Tree], token: ScalaTokenConstants.Value): Unit = tree match {
+      case Some(exp) =>
+        add(token, exp, fromEnd = false)
+        apply(exp)
+      case None =>
+    }
+
+    private def processCases(cases: List[Case]) : Unit = cases.foreach {
+      c =>
+        add(ScalaTokenConstants.Case, c, fromEnd=false)
+        apply(c)
+    }
+
+    private def doMatch(tree: Tree): TR = {
+      tree match {
+        case Term.Do(_) => TR(Some(DoBegin), Some(DoEnd))
+        case Term.Assign(_) => TR(Some(Assign), None)
+        case Term.While(_) => TR(Some(WhileBegin), Some(WhileEnd))
+        case Term.For(_) => TR(Some(ForBegin), Some(ForEnd))
+        case Term.Try(expr, catchp, finallyp) => TR(Some(TryBegin),
+          traverse = _ => {
+            apply(expr)
+            if (catchp.nonEmpty) {
+              val start = catchp.head.pos
+              val end = catchp.last.pos
+              val len = end.endLine - start.startLine
+
+              add(CatchBegin, start.startLine, start.startColumn, len)
+              processCases(catchp)
+              add(CatchEnd, end.endLine, end.endColumn, len)
+            }
+
+            maybeAddAndApply(finallyp, ScalaTokenConstants.Finally)
+          })
+        case Term.TryWithHandler(expr, catchp, finallyp) => TR(Some(TryBegin),
+          traverse = _ => {
+            apply(expr)
+            add(CatchBegin, catchp, fromEnd=false)
+            apply(catchp)
+            add(CatchEnd, catchp, fromEnd = true)
+
+            maybeAddAndApply(finallyp, ScalaTokenConstants.Finally)
+          })
+        case Term.Apply(fun, args) =>
+          // `f()` can also be written as `f`, so we simply don't see `f()` as a function call
+          // But Java style function calls with no args should still be recognized
+          TR(traverse = _ => {
+            add(Apply, tree, fromEnd=false)
+            apply(fun)
+            for (arg <- args) {
+              add(Argument, arg, fromEnd = false)
+              arg match {
+                case Term.Assign(lhs, rhs) =>
+                  apply(lhs)
+                  apply(rhs)
+                case _ => apply(arg)
+              }
+            }
+          })
+        case Term.NewAnonymous(_) => TR(Some(NewCreationBegin), Some(ScalaTokenConstants.NewCreationBegin))
+        case Term.Return(_) => TR(Some(ScalaTokenConstants.Return))
+        case Term.Match(expr, cases) => TR(traverse = _ => {
+          add(MatchBegin, tree, fromEnd=false)
+          apply(expr)
+          processCases(cases)
+          add(MatchEnd, tree, fromEnd=true)
+        })
+        case Term.Throw(_) => TR(Some(Throw))
+        case Term.Function(_) => TR(Some(FunctionBegin), Some(FunctionEnd))
+        case Term.PartialFunction(cases) => TR(traverse = _ => {
+          add(PartialFunctionBegin, tree, fromEnd=false)
+          processCases(cases)
+          add(PartialFunctionEnd, tree, fromEnd=true)
+        })
+        case Term.ForYield(_) => TR(traverse = _ => {
+          add(ForBegin, tree, fromEnd=false)
+          add(Yield, tree, fromEnd = false)
+
+          tree.children.foreach(apply)
+
+          add(ForEnd, tree, fromEnd = true)
+        })
+        case Term.If(cond, thenp, elsep) => TR(traverse = _ => {
+          add(IfBegin, tree, fromEnd=false)
+          apply(cond)
+
+          apply(thenp)
+
+          elsep match {
+            case Lit.Unit() =>
+            case _ =>
+              add(ScalaTokenConstants.Else, elsep, fromEnd=false)
+          }
+          apply(elsep)
+
+          add(IfEnd, tree, fromEnd=true)
+        })
+
+        case scala.meta.Pkg(_) => TR(Some(Package))
+        case scala.meta.Import(_) => TR(Some(ScalaTokenConstants.Import))
+
+        case Defn.Def(_) => TR(Some(MethodBegin), Some(MethodEnd))
+        case Defn.Macro(_) => TR(Some(MacroBegin), Some(MacroEnd))
+        case Defn.Class(_) => TR(Some(ClassBegin), Some(ClassEnd))
+        case Defn.Object(_) => TR(Some(ObjectBegin), Some(ObjectEnd))
+        case Defn.Trait(_) => TR(Some(TraitBegin), Some(TraitEnd))
+        case Defn.Type(_) => TR(Some(ScalaTokenConstants.Type))
+        case Defn.Var(mods, pats, decltype, rhs) => TR(traverse = _ => {
+          apply(mods)
+          for (pat <- pats) {
+            add(VariableDefinition, pat, fromEnd=false)
+            apply(pat)
+            apply(decltype)
+
+            rhs match {
+              case Some(realRhs) =>
+                add(Assign, realRhs, fromEnd=false)
+                apply(realRhs)
+              case None =>
+            }
+          }
+        })
+        case Defn.Val(mods, pats, decltype, rhs) => TR(traverse = _ => {
+          apply(mods)
+          for (pat <- pats) {
+            add(VariableDefinition, pat, fromEnd=false)
+            apply(pat)
+            apply(decltype)
+
+            add(Assign, rhs, fromEnd=false)
+            apply(rhs)
+          }
+
+        })
+
+        case Decl.Var(_) => TR(Some(VariableDefinition))
+        case Decl.Var(_) => TR(Some(VariableDefinition))
+        case Decl.Def(_) => TR(Some(MethodBegin), Some(MethodEnd))
+        case Decl.Type(_) => TR(Some(ScalaTokenConstants.Type))
+
+        case Ctor.Secondary(_) => TR(Some(ConstructorBegin), Some(ConstructorEnd))
+
+        case Enumerator.Guard(_) => TR(Some(Guard))
+
+        case Term.Param(_) => TR(traverse = _ => add(Parameter, tree, fromEnd = false))
+        case Term.NewAnonymous(_) => TR(Some(ClassBegin), Some(ClassEnd))
+        case Term.ApplyInfix(_, op, _, _) if op.value.contains("=") && !op.value.equals("==") =>
+          TR(Some(Assign), None)
+
+        case _ => TR()
+      }
+    }
+
+    override def apply(tree: Tree): Unit = {
+      val res = doMatch(tree)
+
+      res.before match {
+        case Some(value) => add(value, tree, fromEnd = false)
+        case None =>
+      }
+
+      res.traverse(tree)
+
+      res.after match {
+        case Some(value) => add(value, tree, fromEnd = true)
+        case None =>
+      }
+    }
+  }
+
+
+
+  def parse(dir: File, files: Array[String]): TokenList = {
+    tokens = new TokenList
+    errors = 0
+
+    for (file <- files) {
+      currentFile = file
+      // getProgram.print(null, "Parsing file" + file + "\n")
+      if (!parseFile(dir, file)) {
+        errors += 1
+      }
+      System.gc()
+    }
+
+    tokens
+  }
+
+  def parseFile(dir: File, file: String): Boolean = {
+    currentFile = file
+
+    try {
+      val f = new File(dir, file)
+      val bytes = java.nio.file.Files.readAllBytes(f.toPath)
+      val text = new String(bytes, "UTF-8")
+      val input = Input.VirtualFile(f.getPath, text)
+      val ast = input.parse[Source].get
+      traverser(ast)
+
+      add(FileEnd, text.count(_ == '\n') + 1, 0, 0)
+    } catch {
+      case e: Throwable =>
+        e.printStackTrace()
+        return false
+    }
+
+    true
+  }
+
+  private def add(typ: ScalaTokenConstants.Value, line: Int, column: Int, length: Int): Unit = {
+    tokens.addToken(new ScalaToken(typ.id, currentFile, line, column, length))
+  }
+
+
+  private def add(tType: ScalaTokenConstants.Value, node: Tree, fromEnd: Boolean): Unit = {
+    tokens.addToken(new ScalaToken(node, fromEnd, currentFile, tType))
+  }
+}

pom.xml

@@ -50,6 +50,7 @@
         <module>jplag.frontend.java</module>
         <module>jplag.frontend.python-3</module>
         <module>jplag.frontend.rlang</module>
+        <module>jplag.frontend.scala</module>
         <module>jplag.frontend.scheme</module>
         <module>jplag.frontend.text</module>
         <module>jplag</module>
@@ -162,6 +163,11 @@
                 <artifactId>rlang</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>${project.groupId}</groupId>
+                <artifactId>scala</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>${project.groupId}</groupId>
                 <artifactId>cpp</artifactId>