Fix a few frontend issues (#187)

* Use Charset and StandardCharsets in the Unicode reader instead of strings.
* Initialize CPP scanner with UTF-8 encoding.
* Remove deprecated non-null annotation.

jplag.frontend-utils/src/main/java/jplag/UnicodeReader.java

@@ -21,14 +21,16 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.PushbackInputStream;
 import java.io.Reader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 
 /**
  * Generic unicode text reader, which will use BOM mark to identify the encoding to be used. If BOM is not found then
- * use a given default encoding. System default is used if: BOM mark is not found and defaultEnc is NULL.<br />
+ * use a given default encoding. UTF-8 is used if: BOM mark is not found and defaultEnc is NULL.<br />
  * Usage pattern:
  * 
  * <pre>
- * String defaultEnc = "UTF-8"; // or NULL to use system default
+ * String defaultEnc = "UTF-16BE"; // or NULL to use system default
  * FileInputStream fis = new FileInputStream(file);
  * Reader in = new UnicodeReader(fis, defaultEnc);
  * </pre>
@@ -38,16 +40,16 @@ public class UnicodeReader extends Reader {
 
     private InputStreamReader internalIn2 = null;
 
-    private String defaultEnc;
+    private Charset defaultEnc;
 
     private static final int BOM_SIZE = 4;
 
-    public UnicodeReader(InputStream in, String defaultEnc) {
+    public UnicodeReader(InputStream in, Charset defaultEnc) {
         internalIn = new PushbackInputStream(in, BOM_SIZE);
         this.defaultEnc = defaultEnc;
     }
 
-    public String getDefaultEncoding() {
+    public Charset getDefaultEncoding() {
         return defaultEnc;
     }
 
@@ -64,25 +66,25 @@ public class UnicodeReader extends Reader {
         if (internalIn2 != null)
             return;
 
-        String encoding;
+        Charset encoding;
         byte bom[] = new byte[BOM_SIZE];
         int n, unread;
         n = internalIn.read(bom, 0, bom.length);
 
         if ((bom[0] == (byte) 0xEF) && (bom[1] == (byte) 0xBB) && (bom[2] == (byte) 0xBF)) {
-            encoding = "UTF-8";
+            encoding = StandardCharsets.UTF_8;
             unread = n - 3;
         } else if ((bom[0] == (byte) 0xFE) && (bom[1] == (byte) 0xFF)) {
-            encoding = "UTF-16BE";
+            encoding = StandardCharsets.UTF_16BE;
             unread = n - 2;
         } else if ((bom[0] == (byte) 0xFF) && (bom[1] == (byte) 0xFE)) {
-            encoding = "UTF-16LE";
+            encoding = StandardCharsets.UTF_16LE;
             unread = n - 2;
         } else if ((bom[0] == (byte) 0x00) && (bom[1] == (byte) 0x00) && (bom[2] == (byte) 0xFE) && (bom[3] == (byte) 0xFF)) {
-            encoding = "UTF-32BE";
+            encoding = Charset.forName("UTF-32BE");
             unread = n - 4;
         } else if ((bom[0] == (byte) 0xFF) && (bom[1] == (byte) 0xFE) && (bom[2] == (byte) 0x00) && (bom[3] == (byte) 0x00)) {
-            encoding = "UTF-32LE";
+            encoding = Charset.forName("UTF-32LE");
             unread = n - 4;
         } else {
             // Unicode BOM mark not found, unread all bytes
@@ -95,7 +97,7 @@ public class UnicodeReader extends Reader {
 
         // Use given encoding
         if (encoding == null) {
-            internalIn2 = new InputStreamReader(internalIn);
+            internalIn2 = new InputStreamReader(internalIn, StandardCharsets.UTF_8);
         } else {
             internalIn2 = new InputStreamReader(internalIn, encoding);
         }

jplag.frontend.cpp/src/main/javacc/CPP.jj

@@ -21,13 +21,12 @@ import jplag.*;
 public class CPPScanner implements CPPTokenConstants {
     private static Scanner scanner2;
     public static boolean scanFile(File dir, String fileName,
-        jplag.cpp.CPPScanner scanner, jplag.cpp.Scanner scannerX)
-    {
+        jplag.cpp.CPPScanner scanner, jplag.cpp.Scanner scannerX) {
         try {
             InputStream in = new NewlineStream(
                 new FileInputStream(new File(dir,fileName)));
               if (scanner==null) {
-                scanner = new CPPScanner(in);
+                scanner = new CPPScanner(in, "UTF-8");
               } else {
                 scanner.ReInit(in);
               }

jplag.frontend.csharp-1.2/src/main/java/jplag/csharp/Parser.java

@@ -2,6 +2,7 @@ package jplag.csharp;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.nio.charset.StandardCharsets;
 
 import jplag.TokenList;
 import jplag.UnicodeReader;
@@ -30,7 +31,7 @@ public class Parser extends jplag.Parser implements CSharpTokenConstants {
 			FileInputStream fis = new FileInputStream(new File(dir, file));
 			currentFile = file;
 			// Create a scanner that reads from the input stream passed to us
-			CSharpLexer lexer = new CSharpLexer(new UnicodeReader(fis, "UTF-8"));
+			CSharpLexer lexer = new CSharpLexer(new UnicodeReader(fis, StandardCharsets.UTF_8));
 			lexer.setFilename(file);
 			lexer.setTabSize(1);
 

jplag.frontend.java-1.5/src/main/javacc/Java1.5.jj

@@ -34,6 +34,7 @@ package jplag.java15.grammar;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.nio.charset.StandardCharsets;
 
 import jplag.UnicodeReader;
 import jplag.java15.JavaTokenConstants;
@@ -137,11 +138,11 @@ public class JavaParser implements JavaTokenConstants
         try {
             FileInputStream in = new FileInputStream(new File(dir,fileName));
             if (parser == null) {
-                JavaTabModCharStream charStream = new JavaTabModCharStream(new UnicodeReader(in, "UTF-8"));
+                JavaTabModCharStream charStream = new JavaTabModCharStream(new UnicodeReader(in, StandardCharsets.UTF_8));
                 parser = new JavaParser(charStream);
                 parser.charStream = charStream;
             } else {
-                parser.charStream.ReInit(new UnicodeReader(in, "UTF-8"));
+                parser.charStream.ReInit(new UnicodeReader(in, StandardCharsets.UTF_8));
                 parser.ReInit(parser.charStream);
             }
             parser.parser2 = parserX;

jplag.frontend.python-3/src/main/java/jplag/python3/JplagPython3Listener.java

@@ -1,7 +1,6 @@
 package jplag.python3;
 
 import org.antlr.v4.runtime.ParserRuleContext;
-import org.antlr.v4.runtime.misc.NotNull;
 import org.antlr.v4.runtime.tree.ErrorNode;
 import org.antlr.v4.runtime.tree.TerminalNode;
 
@@ -17,415 +16,415 @@ public class JplagPython3Listener implements Python3Listener, Python3TokenConsta
     }
 
     @Override
-    public void enterTestlist(@NotNull Python3Parser.TestlistContext ctx) {
+    public void enterTestlist(Python3Parser.TestlistContext ctx) {
     }
 
     @Override
-    public void exitTestlist(@NotNull Python3Parser.TestlistContext ctx) {
+    public void exitTestlist(Python3Parser.TestlistContext ctx) {
     }
 
     @Override
-    public void enterAssert_stmt(@NotNull Python3Parser.Assert_stmtContext ctx) {
+    public void enterAssert_stmt(Python3Parser.Assert_stmtContext ctx) {
         jplagParser.add(ASSERT, ctx.getStart());
     }
 
     @Override
-    public void exitAssert_stmt(@NotNull Python3Parser.Assert_stmtContext ctx) {
+    public void exitAssert_stmt(Python3Parser.Assert_stmtContext ctx) {
     }
 
     @Override
-    public void enterArgument(@NotNull Python3Parser.ArgumentContext ctx) {
+    public void enterArgument(Python3Parser.ArgumentContext ctx) {
     }
 
     @Override
-    public void exitArgument(@NotNull Python3Parser.ArgumentContext ctx) {
+    public void exitArgument(Python3Parser.ArgumentContext ctx) {
     }
 
     @Override
-    public void enterNot_test(@NotNull Python3Parser.Not_testContext ctx) {
+    public void enterNot_test(Python3Parser.Not_testContext ctx) {
     }
 
     @Override
-    public void exitNot_test(@NotNull Python3Parser.Not_testContext ctx) {
+    public void exitNot_test(Python3Parser.Not_testContext ctx) {
     }
 
     @Override
-    public void enterFile_input(@NotNull Python3Parser.File_inputContext ctx) {
+    public void enterFile_input(Python3Parser.File_inputContext ctx) {
     }
 
     @Override
-    public void exitFile_input(@NotNull Python3Parser.File_inputContext ctx) {
+    public void exitFile_input(Python3Parser.File_inputContext ctx) {
     }
 
     @Override
-    public void enterXor_expr(@NotNull Python3Parser.Xor_exprContext ctx) {
+    public void enterXor_expr(Python3Parser.Xor_exprContext ctx) {
     }
 
     @Override
-    public void exitXor_expr(@NotNull Python3Parser.Xor_exprContext ctx) {
+    public void exitXor_expr(Python3Parser.Xor_exprContext ctx) {
     }
 
     @Override
-    public void enterImport_from(@NotNull Python3Parser.Import_fromContext ctx) {
+    public void enterImport_from(Python3Parser.Import_fromContext ctx) {
     }
 
     @Override
-    public void exitImport_from(@NotNull Python3Parser.Import_fromContext ctx) {
+    public void exitImport_from(Python3Parser.Import_fromContext ctx) {
     }
 
     @Override
-    public void enterSingle_input(@NotNull Python3Parser.Single_inputContext ctx) {
+    public void enterSingle_input(Python3Parser.Single_inputContext ctx) {
     }
 
     @Override
-    public void exitSingle_input(@NotNull Python3Parser.Single_inputContext ctx) {
+    public void exitSingle_input(Python3Parser.Single_inputContext ctx) {
     }
 
     @Override
-    public void enterDecorated(@NotNull Python3Parser.DecoratedContext ctx) {
+    public void enterDecorated(Python3Parser.DecoratedContext ctx) {
         jplagParser.add(DEC_BEGIN, ctx.getStart());
     }
 
     @Override
-    public void exitDecorated(@NotNull Python3Parser.DecoratedContext ctx) {
+    public void exitDecorated(Python3Parser.DecoratedContext ctx) {
         jplagParser.addEnd(DEC_END, ctx.getStart());
     }
 
     @Override
-    public void enterWith_item(@NotNull Python3Parser.With_itemContext ctx) {
+    public void enterWith_item(Python3Parser.With_itemContext ctx) {
     }
 
     @Override
-    public void exitWith_item(@NotNull Python3Parser.With_itemContext ctx) {
+    public void exitWith_item(Python3Parser.With_itemContext ctx) {
     }
 
     @Override
-    public void enterRaise_stmt(@NotNull Python3Parser.Raise_stmtContext ctx) {
+    public void enterRaise_stmt(Python3Parser.Raise_stmtContext ctx) {
         jplagParser.add(RAISE, ctx.getStart());
     }
 
     @Override
-    public void exitRaise_stmt(@NotNull Python3Parser.Raise_stmtContext ctx) {
+    public void exitRaise_stmt(Python3Parser.Raise_stmtContext ctx) {
     }
 
     @Override
-    public void enterImport_as_name(@NotNull Python3Parser.Import_as_nameContext ctx) {
+    public void enterImport_as_name(Python3Parser.Import_as_nameContext ctx) {
     }
 
     @Override
-    public void exitImport_as_name(@NotNull Python3Parser.Import_as_nameContext ctx) {
+    public void exitImport_as_name(Python3Parser.Import_as_nameContext ctx) {
     }
 
     @Override
-    public void enterExcept_clause(@NotNull Python3Parser.Except_clauseContext ctx) {
+    public void enterExcept_clause(Python3Parser.Except_clauseContext ctx) {
         jplagParser.add(EXCEPT_BEGIN, ctx.getStart());
     }
 
     @Override
-    public void exitExcept_clause(@NotNull Python3Parser.Except_clauseContext ctx) {
+    public void exitExcept_clause(Python3Parser.Except_clauseContext ctx) {
         jplagParser.addEnd(EXCEPT_END, ctx.getStart());
     }
 
     @Override
-    public void enterCompound_stmt(@NotNull Python3Parser.Compound_stmtContext ctx) {
+    public void enterCompound_stmt(Python3Parser.Compound_stmtContext ctx) {
     }
 
     @Override
-    public void exitCompound_stmt(@NotNull Python3Parser.Compound_stmtContext ctx) {
+    public void exitCompound_stmt(Python3Parser.Compound_stmtContext ctx) {
     }
 
     @Override
-    public void enterAnd_expr(@NotNull Python3Parser.And_exprContext ctx) {
+    public void enterAnd_expr(Python3Parser.And_exprContext ctx) {
     }
 
     @Override
-    public void exitAnd_expr(@NotNull Python3Parser.And_exprContext ctx) {
+    public void exitAnd_expr(Python3Parser.And_exprContext ctx) {
     }
 
     @Override
-    public void enterLambdef_nocond(@NotNull Python3Parser.Lambdef_nocondContext ctx) {
+    public void enterLambdef_nocond(Python3Parser.Lambdef_nocondContext ctx) {
     }
 
     @Override
-    public void exitLambdef_nocond(@NotNull Python3Parser.Lambdef_nocondContext ctx) {
+    public void exitLambdef_nocond(Python3Parser.Lambdef_nocondContext ctx) {
     }
 
     @Override
-    public void enterDictorsetmaker(@NotNull Python3Parser.DictorsetmakerContext ctx) {
+    public void enterDictorsetmaker(Python3Parser.DictorsetmakerContext ctx) {
         jplagParser.add(ARRAY, ctx.getStart());
     }
 
     @Override
-    public void exitDictorsetmaker(@NotNull Python3Parser.DictorsetmakerContext ctx) {
+    public void exitDictorsetmaker(Python3Parser.DictorsetmakerContext ctx) {
     }
 
     @Override
-    public void enterReturn_stmt(@NotNull Python3Parser.Return_stmtContext ctx) {
+    public void enterReturn_stmt(Python3Parser.Return_stmtContext ctx) {
         jplagParser.add(RETURN, ctx.getStart());
     }
 
     @Override
-    public void exitReturn_stmt(@NotNull Python3Parser.Return_stmtContext ctx) {
+    public void exitReturn_stmt(Python3Parser.Return_stmtContext ctx) {
     }
 
     @Override
-    public void enterDotted_name(@NotNull Python3Parser.Dotted_nameContext ctx) {
+    public void enterDotted_name(Python3Parser.Dotted_nameContext ctx) {
     }
 
     @Override
-    public void exitDotted_name(@NotNull Python3Parser.Dotted_nameContext ctx) {
+    public void exitDotted_name(Python3Parser.Dotted_nameContext ctx) {
     }
 
     @Override
-    public void enterFlow_stmt(@NotNull Python3Parser.Flow_stmtContext ctx) {
+    public void enterFlow_stmt(Python3Parser.Flow_stmtContext ctx) {
     }
 
     @Override
-    public void exitFlow_stmt(@NotNull Python3Parser.Flow_stmtContext ctx) {
+    public void exitFlow_stmt(Python3Parser.Flow_stmtContext ctx) {
     }
 
     @Override
-    public void enterWhile_stmt(@NotNull Python3Parser.While_stmtContext ctx) {
+    public void enterWhile_stmt(Python3Parser.While_stmtContext ctx) {
         jplagParser.add(WHILE_BEGIN, ctx.getStart());
     }
 
     @Override
-    public void exitWhile_stmt(@NotNull Python3Parser.While_stmtContext ctx) {
+    public void exitWhile_stmt(Python3Parser.While_stmtContext ctx) {
         jplagParser.addEnd(WHILE_END, ctx.getStart());
     }
 
     @Override
-    public void enterOr_test(@NotNull Python3Parser.Or_testContext ctx) {
+    public void enterOr_test(Python3Parser.Or_testContext ctx) {
     }
 
     @Override
-    public void exitOr_test(@NotNull Python3Parser.Or_testContext ctx) {
+    public void exitOr_test(Python3Parser.Or_testContext ctx) {
     }
 
     @Override
-    public void enterComparison(@NotNull Python3Parser.ComparisonContext ctx) {
+    public void enterComparison(Python3Parser.ComparisonContext ctx) {
     }
 
     @Override
-    public void exitComparison(@NotNull Python3Parser.ComparisonContext ctx) {
+    public void exitComparison(Python3Parser.ComparisonContext ctx) {
     }
 
     @Override
-    public void enterTest(@NotNull Python3Parser.TestContext ctx) {
+    public void enterTest(Python3Parser.TestContext ctx) {
     }
 
     @Override
-    public void exitTest(@NotNull Python3Parser.TestContext ctx) {
+    public void exitTest(Python3Parser.TestContext ctx) {
     }
 
     @Override
-    public void enterSubscript(@NotNull Python3Parser.SubscriptContext ctx) {
+    public void enterSubscript(Python3Parser.SubscriptContext ctx) {
     }
 
     @Override
-    public void exitSubscript(@NotNull Python3Parser.SubscriptContext ctx) {
+    public void exitSubscript(Python3Parser.SubscriptContext ctx) {
     }
 
     @Override
-    public void enterComp_for(@NotNull Python3Parser.Comp_forContext ctx) {
+    public void enterComp_for(Python3Parser.Comp_forContext ctx) {
     }
 
     @Override
-    public void exitComp_for(@NotNull Python3Parser.Comp_forContext ctx) {
+    public void exitComp_for(Python3Parser.Comp_forContext ctx) {
     }
 
     @Override
-    public void enterYield_arg(@NotNull Python3Parser.Yield_argContext ctx) {
+    public void enterYield_arg(Python3Parser.Yield_argContext ctx) {
         jplagParser.add(YIELD, ctx.getStart());
     }
 
     @Override
-    public void exitYield_arg(@NotNull Python3Parser.Yield_argContext ctx) {
+    public void exitYield_arg(Python3Parser.Yield_argContext ctx) {
     }
 
     @Override
-    public void enterYield_expr(@NotNull Python3Parser.Yield_exprContext ctx) {
+    public void enterYield_expr(Python3Parser.Yield_exprContext ctx) {
     }
 
     @Override
-    public void exitYield_expr(@NotNull Python3Parser.Yield_exprContext ctx) {
+    public void exitYield_expr(Python3Parser.Yield_exprContext ctx) {
     }
 
     @Override
-    public void enterImport_stmt(@NotNull Python3Parser.Import_stmtContext ctx) {
+    public void enterImport_stmt(Python3Parser.Import_stmtContext ctx) {
         jplagParser.add(IMPORT, ctx.getStart());
     }
 
     @Override
-    public void exitImport_stmt(@NotNull Python3Parser.Import_stmtContext ctx) {
+    public void exitImport_stmt(Python3Parser.Import_stmtContext ctx) {
     }
 
     @Override
-    public void enterShift_expr(@NotNull Python3Parser.Shift_exprContext ctx) {
+    public void enterShift_expr(Python3Parser.Shift_exprContext ctx) {
     }
 
     @Override
-    public void exitShift_expr(@NotNull Python3Parser.Shift_exprContext ctx) {
+    public void exitShift_expr(Python3Parser.Shift_exprContext ctx) {
     }
 
     @Override
-    public void enterLambdef(@NotNull Python3Parser.LambdefContext ctx) {
+    public void enterLambdef(Python3Parser.LambdefContext ctx) {
         jplagParser.add(LAMBDA, ctx.getStart());
     }
 
     @Override
-    public void exitLambdef(@NotNull Python3Parser.LambdefContext ctx) {
+    public void exitLambdef(Python3Parser.LambdefContext ctx) {
     }
 
     @Override
-    public void enterAnd_test(@NotNull Python3Parser.And_testContext ctx) {
+    public void enterAnd_test(Python3Parser.And_testContext ctx) {
     }
 
     @Override
-    public void exitAnd_test(@NotNull Python3Parser.And_testContext ctx) {
+    public void exitAnd_test(Python3Parser.And_testContext ctx) {
     }
 
     @Override
-    public void enterGlobal_stmt(@NotNull Python3Parser.Global_stmtContext ctx) {
+    public void enterGlobal_stmt(Python3Parser.Global_stmtContext ctx) {
     }
 
     @Override
-    public void exitGlobal_stmt(@NotNull Python3Parser.Global_stmtContext ctx) {
+    public void exitGlobal_stmt(Python3Parser.Global_stmtContext ctx) {
     }
 
     @Override
-    public void enterImport_as_names(@NotNull Python3Parser.Import_as_namesContext ctx) {
+    public void enterImport_as_names(Python3Parser.Import_as_namesContext ctx) {
     }
 
     @Override
-    public void exitImport_as_names(@NotNull Python3Parser.Import_as_namesContext ctx) {
+    public void exitImport_as_names(Python3Parser.Import_as_namesContext ctx) {
     }
 
     @Override
-    public void enterDecorators(@NotNull Python3Parser.DecoratorsContext ctx) {
+    public void enterDecorators(Python3Parser.DecoratorsContext ctx) {
     }
 
     @Override
-    public void exitDecorators(@NotNull Python3Parser.DecoratorsContext ctx) {
+    public void exitDecorators(Python3Parser.DecoratorsContext ctx) {
     }
 
     @Override
-    public void enterTry_stmt(@NotNull Python3Parser.Try_stmtContext ctx) {
+    public void enterTry_stmt(Python3Parser.Try_stmtContext ctx) {
         jplagParser.add(TRY_BEGIN, ctx.getStart());
     }
 
     @Override
-    public void exitTry_stmt(@NotNull Python3Parser.Try_stmtContext ctx) {
+    public void exitTry_stmt(Python3Parser.Try_stmtContext ctx) {
     }
 
     @Override
-    public void enterComp_op(@NotNull Python3Parser.Comp_opContext ctx) {
+    public void enterComp_op(Python3Parser.Comp_opContext ctx) {
     }
 
     @Override
-    public void exitComp_op(@NotNull Python3Parser.Comp_opContext ctx) {
+    public void exitComp_op(Python3Parser.Comp_opContext ctx) {
     }
 
     @Override
-    public void enterStar_expr(@NotNull Python3Parser.Star_exprContext ctx) {
+    public void enterStar_expr(Python3Parser.Star_exprContext ctx) {
     }
 
     @Override
-    public void exitStar_expr(@NotNull Python3Parser.Star_exprContext ctx) {
+    public void exitStar_expr(Python3Parser.Star_exprContext ctx) {
     }
 
     @Override
-    public void enterBreak_stmt(@NotNull Python3Parser.Break_stmtContext ctx) {
+    public void enterBreak_stmt(Python3Parser.Break_stmtContext ctx) {
         jplagParser.add(BREAK, ctx.getStart());
     }
 
     @Override
-    public void exitBreak_stmt(@NotNull Python3Parser.Break_stmtContext ctx) {
+    public void exitBreak_stmt(Python3Parser.Break_stmtContext ctx) {
     }
 
     @Override
-    public void enterParameters(@NotNull Python3Parser.ParametersContext ctx) {
+    public void enterParameters(Python3Parser.ParametersContext ctx) {
     }
 
     @Override
-    public void exitParameters(@NotNull Python3Parser.ParametersContext ctx) {
+    public void exitParameters(Python3Parser.ParametersContext ctx) {
     }
 
     @Override
-    public void enterDecorator(@NotNull Python3Parser.DecoratorContext ctx) {
+    public void enterDecorator(Python3Parser.DecoratorContext ctx) {
     }
 
     @Override
-    public void exitDecorator(@NotNull Python3Parser.DecoratorContext ctx) {
+    public void exitDecorator(Python3Parser.DecoratorContext ctx) {
     }
 
     @Override
-    public void enterTfpdef(@NotNull Python3Parser.TfpdefContext ctx) {
+    public void enterTfpdef(Python3Parser.TfpdefContext ctx) {
     }
 
     @Override
-    public void exitTfpdef(@NotNull Python3Parser.TfpdefContext ctx) {
+    public void exitTfpdef(Python3Parser.TfpdefContext ctx) {
     }
 
     @Override
-    public void enterTestlist_comp(@NotNull Python3Parser.Testlist_compContext ctx) {
+    public void enterTestlist_comp(Python3Parser.Testlist_compContext ctx) {
         if (ctx.getText().contains(",")) {
             jplagParser.add(ARRAY, ctx.getStart());
         }
     }
 
     @Override
-    public void exitTestlist_comp(@NotNull Python3Parser.Testlist_compContext ctx) {
+    public void exitTestlist_comp(Python3Parser.Testlist_compContext ctx) {
     }
 
     @Override
-    public void enterIf_stmt(@NotNull Python3Parser.If_stmtContext ctx) {
+    public void enterIf_stmt(Python3Parser.If_stmtContext ctx) {
         jplagParser.add(IF_BEGIN, ctx.getStart());
     }
 
     @Override
-    public void exitIf_stmt(@NotNull Python3Parser.If_stmtContext ctx) {
+    public void exitIf_stmt(Python3Parser.If_stmtContext ctx) {
         jplagParser.addEnd(IF_END, ctx.getStart());
     }
 
     @Override
-    public void enterWith_stmt(@NotNull Python3Parser.With_stmtContext ctx) {
+    public void enterWith_stmt(Python3Parser.With_stmtContext ctx) {
         jplagParser.add(WITH_BEGIN, ctx.getStart());
     }
 
     @Override
-    public void exitWith_stmt(@NotNull Python3Parser.With_stmtContext ctx) {
+    public void exitWith_stmt(Python3Parser.With_stmtContext ctx) {
         jplagParser.addEnd(WITH_END, ctx.getStart());
     }
 
     @Override
-    public void enterClassdef(@NotNull Python3Parser.ClassdefContext ctx) {
+    public void enterClassdef(Python3Parser.ClassdefContext ctx) {
         jplagParser.add(CLASS_BEGIN, ctx.getStart());
     }
 
     @Override
-    public void exitClassdef(@NotNull Python3Parser.ClassdefContext ctx) {
+    public void exitClassdef(Python3Parser.ClassdefContext ctx) {
         jplagParser.addEnd(CLASS_END, ctx.getStart());
     }
 
     @Override
-    public void enterExprlist(@NotNull Python3Parser.ExprlistContext ctx) {
+    public void enterExprlist(Python3Parser.ExprlistContext ctx) {
     }
 
     @Override
-    public void exitExprlist(@NotNull Python3Parser.ExprlistContext ctx) {
+    public void exitExprlist(Python3Parser.ExprlistContext ctx) {
     }
 
     @Override
-    public void enterSmall_stmt(@NotNull Python3Parser.Small_stmtContext ctx) {
+    public void enterSmall_stmt(Python3Parser.Small_stmtContext ctx) {
     }
 
     @Override
-    public void exitSmall_stmt(@NotNull Python3Parser.Small_stmtContext ctx) {
+    public void exitSmall_stmt(Python3Parser.Small_stmtContext ctx) {
     }
 
     @Override
-    public void enterTrailer(@NotNull Python3Parser.TrailerContext ctx) {
+    public void enterTrailer(Python3Parser.TrailerContext ctx) {
         if (ctx.getText().charAt(0)=='(') {
             jplagParser.add(APPLY, ctx.getStart());
         } else {
@@ -434,283 +433,283 @@ public class JplagPython3Listener implements Python3Listener, Python3TokenConsta
     }
 
     @Override
-    public void exitTrailer(@NotNull Python3Parser.TrailerContext ctx) {
+    public void exitTrailer(Python3Parser.TrailerContext ctx) {
     }
 
     @Override
-    public void enterDotted_as_names(@NotNull Python3Parser.Dotted_as_namesContext ctx) {
+    public void enterDotted_as_names(Python3Parser.Dotted_as_namesContext ctx) {
     }
 
     @Override
-    public void exitDotted_as_names(@NotNull Python3Parser.Dotted_as_namesContext ctx) {
+    public void exitDotted_as_names(Python3Parser.Dotted_as_namesContext ctx) {
     }
 
     @Override
-    public void enterArith_expr(@NotNull Python3Parser.Arith_exprContext ctx) {
+    public void enterArith_expr(Python3Parser.Arith_exprContext ctx) {
     }
 
     @Override
-    public void exitArith_expr(@NotNull Python3Parser.Arith_exprContext ctx) {
+    public void exitArith_expr(Python3Parser.Arith_exprContext ctx) {
     }
 
     @Override
-    public void enterArglist(@NotNull Python3Parser.ArglistContext ctx) {
+    public void enterArglist(Python3Parser.ArglistContext ctx) {
     }
 
     @Override
-    public void exitArglist(@NotNull Python3Parser.ArglistContext ctx) {
+    public void exitArglist(Python3Parser.ArglistContext ctx) {
     }
 
     @Override
-    public void enterSimple_stmt(@NotNull Python3Parser.Simple_stmtContext ctx) {
+    public void enterSimple_stmt(Python3Parser.Simple_stmtContext ctx) {
     }
 
     @Override
-    public void exitSimple_stmt(@NotNull Python3Parser.Simple_stmtContext ctx) {
+    public void exitSimple_stmt(Python3Parser.Simple_stmtContext ctx) {
     }
 
     @Override
-    public void enterTypedargslist(@NotNull Python3Parser.TypedargslistContext ctx) {
+    public void enterTypedargslist(Python3Parser.TypedargslistContext ctx) {
     }
 
     @Override
-    public void exitTypedargslist(@NotNull Python3Parser.TypedargslistContext ctx) {
+    public void exitTypedargslist(Python3Parser.TypedargslistContext ctx) {
     }
 
     @Override
-    public void enterExpr(@NotNull Python3Parser.ExprContext ctx) {
+    public void enterExpr(Python3Parser.ExprContext ctx) {
     }
 
     @Override
-    public void exitExpr(@NotNull Python3Parser.ExprContext ctx) {
+    public void exitExpr(Python3Parser.ExprContext ctx) {
     }
 
     @Override
-    public void enterTerm(@NotNull Python3Parser.TermContext ctx) {
+    public void enterTerm(Python3Parser.TermContext ctx) {
     }
 
     @Override
-    public void exitTerm(@NotNull Python3Parser.TermContext ctx) {
+    public void exitTerm(Python3Parser.TermContext ctx) {
     }
 
     @Override
-    public void enterPower(@NotNull Python3Parser.PowerContext ctx) {
+    public void enterPower(Python3Parser.PowerContext ctx) {
     }
 
     @Override
-    public void exitPower(@NotNull Python3Parser.PowerContext ctx) {
+    public void exitPower(Python3Parser.PowerContext ctx) {
     }
 
     @Override
-    public void enterDotted_as_name(@NotNull Python3Parser.Dotted_as_nameContext ctx) {
+    public void enterDotted_as_name(Python3Parser.Dotted_as_nameContext ctx) {
     }
 
     @Override
-    public void exitDotted_as_name(@NotNull Python3Parser.Dotted_as_nameContext ctx) {
+    public void exitDotted_as_name(Python3Parser.Dotted_as_nameContext ctx) {
     }
 
     @Override
-    public void enterFactor(@NotNull Python3Parser.FactorContext ctx) {
+    public void enterFactor(Python3Parser.FactorContext ctx) {
     }
 
     @Override
-    public void exitFactor(@NotNull Python3Parser.FactorContext ctx) {
+    public void exitFactor(Python3Parser.FactorContext ctx) {
     }
 
     @Override
-    public void enterSliceop(@NotNull Python3Parser.SliceopContext ctx) {
+    public void enterSliceop(Python3Parser.SliceopContext ctx) {
     }
 
     @Override
-    public void exitSliceop(@NotNull Python3Parser.SliceopContext ctx) {
+    public void exitSliceop(Python3Parser.SliceopContext ctx) {
     }
 
     @Override
-    public void enterFuncdef(@NotNull Python3Parser.FuncdefContext ctx) {
+    public void enterFuncdef(Python3Parser.FuncdefContext ctx) {
         jplagParser.add(METHOD_BEGIN, ctx.getStart());
     }
 
     @Override
-    public void exitFuncdef(@NotNull Python3Parser.FuncdefContext ctx) {
+    public void exitFuncdef(Python3Parser.FuncdefContext ctx) {
         jplagParser.addEnd(METHOD_END, ctx.getStart());
     }
 
     @Override
-    public void enterSubscriptlist(@NotNull Python3Parser.SubscriptlistContext ctx) {
+    public void enterSubscriptlist(Python3Parser.SubscriptlistContext ctx) {
     }
 
     @Override
-    public void exitSubscriptlist(@NotNull Python3Parser.SubscriptlistContext ctx) {
+    public void exitSubscriptlist(Python3Parser.SubscriptlistContext ctx) {
     }
 
     @Override
-    public void enterTest_nocond(@NotNull Python3Parser.Test_nocondContext ctx) {
+    public void enterTest_nocond(Python3Parser.Test_nocondContext ctx) {
     }
 
     @Override
-    public void exitTest_nocond(@NotNull Python3Parser.Test_nocondContext ctx) {
+    public void exitTest_nocond(Python3Parser.Test_nocondContext ctx) {
     }
 
     @Override
-    public void enterComp_iter(@NotNull Python3Parser.Comp_iterContext ctx) {
+    public void enterComp_iter(Python3Parser.Comp_iterContext ctx) {
     }
 
     @Override
-    public void exitComp_iter(@NotNull Python3Parser.Comp_iterContext ctx) {
+    public void exitComp_iter(Python3Parser.Comp_iterContext ctx) {
     }
 
     @Override
-    public void enterNonlocal_stmt(@NotNull Python3Parser.Nonlocal_stmtContext ctx) {
+    public void enterNonlocal_stmt(Python3Parser.Nonlocal_stmtContext ctx) {
     }
 
     @Override
-    public void exitNonlocal_stmt(@NotNull Python3Parser.Nonlocal_stmtContext ctx) {
+    public void exitNonlocal_stmt(Python3Parser.Nonlocal_stmtContext ctx) {
     }
 
     @Override
-    public void enterEval_input(@NotNull Python3Parser.Eval_inputContext ctx) {
+    public void enterEval_input(Python3Parser.Eval_inputContext ctx) {
     }
 
     @Override
-    public void exitEval_input(@NotNull Python3Parser.Eval_inputContext ctx) {
+    public void exitEval_input(Python3Parser.Eval_inputContext ctx) {
     }
 
     @Override
-    public void enterVfpdef(@NotNull Python3Parser.VfpdefContext ctx) {
+    public void enterVfpdef(Python3Parser.VfpdefContext ctx) {
     }
 
     @Override
-    public void exitVfpdef(@NotNull Python3Parser.VfpdefContext ctx) {
+    public void exitVfpdef(Python3Parser.VfpdefContext ctx) {
     }
 
     @Override
-    public void enterImport_name(@NotNull Python3Parser.Import_nameContext ctx) {
+    public void enterImport_name(Python3Parser.Import_nameContext ctx) {
     }
 
     @Override
-    public void exitImport_name(@NotNull Python3Parser.Import_nameContext ctx) {
+    public void exitImport_name(Python3Parser.Import_nameContext ctx) {
     }
 
     @Override
-    public void enterComp_if(@NotNull Python3Parser.Comp_ifContext ctx) {
+    public void enterComp_if(Python3Parser.Comp_ifContext ctx) {
     }
 
     @Override
-    public void exitComp_if(@NotNull Python3Parser.Comp_ifContext ctx) {
+    public void exitComp_if(Python3Parser.Comp_ifContext ctx) {
     }
 
     @Override
-    public void enterAugassign(@NotNull Python3Parser.AugassignContext ctx) {
+    public void enterAugassign(Python3Parser.AugassignContext ctx) {
         jplagParser.add(ASSIGN, ctx.getStart());
     }
 
     @Override
-    public void exitAugassign(@NotNull Python3Parser.AugassignContext ctx) {
+    public void exitAugassign(Python3Parser.AugassignContext ctx) {
     }
 
     @Override
-    public void enterPass_stmt(@NotNull Python3Parser.Pass_stmtContext ctx) {
+    public void enterPass_stmt(Python3Parser.Pass_stmtContext ctx) {
     }
 
     @Override
-    public void exitPass_stmt(@NotNull Python3Parser.Pass_stmtContext ctx) {
+    public void exitPass_stmt(Python3Parser.Pass_stmtContext ctx) {
     }
 
     @Override
-    public void enterExpr_stmt(@NotNull Python3Parser.Expr_stmtContext ctx) {
+    public void enterExpr_stmt(Python3Parser.Expr_stmtContext ctx) {
     }
 
     @Override
-    public void exitExpr_stmt(@NotNull Python3Parser.Expr_stmtContext ctx) {
+    public void exitExpr_stmt(Python3Parser.Expr_stmtContext ctx) {
     }
 
     @Override
-    public void enterYield_stmt(@NotNull Python3Parser.Yield_stmtContext ctx) {
+    public void enterYield_stmt(Python3Parser.Yield_stmtContext ctx) {
         jplagParser.add(YIELD, ctx.getStart());
     }
 
     @Override
-    public void exitYield_stmt(@NotNull Python3Parser.Yield_stmtContext ctx) {
+    public void exitYield_stmt(Python3Parser.Yield_stmtContext ctx) {
     }
 
     @Override
-    public void enterSuite(@NotNull Python3Parser.SuiteContext ctx) {
+    public void enterSuite(Python3Parser.SuiteContext ctx) {
     }
 
     @Override
-    public void exitSuite(@NotNull Python3Parser.SuiteContext ctx) {
+    public void exitSuite(Python3Parser.SuiteContext ctx) {
     }
 
     @Override
-    public void enterContinue_stmt(@NotNull Python3Parser.Continue_stmtContext ctx) {
+    public void enterContinue_stmt(Python3Parser.Continue_stmtContext ctx) {
         jplagParser.add(CONTINUE, ctx.getStart());
     }
 
     @Override
-    public void exitContinue_stmt(@NotNull Python3Parser.Continue_stmtContext ctx) {
+    public void exitContinue_stmt(Python3Parser.Continue_stmtContext ctx) {
     }
 
     @Override
-    public void enterTestlist_star_expr(@NotNull Python3Parser.Testlist_star_exprContext ctx) {
+    public void enterTestlist_star_expr(Python3Parser.Testlist_star_exprContext ctx) {
     }
 
     @Override
-    public void exitTestlist_star_expr(@NotNull Python3Parser.Testlist_star_exprContext ctx) {
+    public void exitTestlist_star_expr(Python3Parser.Testlist_star_exprContext ctx) {
     }
 
     @Override
-    public void enterVarargslist(@NotNull Python3Parser.VarargslistContext ctx) {
+    public void enterVarargslist(Python3Parser.VarargslistContext ctx) {
     }
 
     @Override
-    public void exitVarargslist(@NotNull Python3Parser.VarargslistContext ctx) {
+    public void exitVarargslist(Python3Parser.VarargslistContext ctx) {
     }
 
     @Override
-    public void enterFor_stmt(@NotNull Python3Parser.For_stmtContext ctx) {
+    public void enterFor_stmt(Python3Parser.For_stmtContext ctx) {
         jplagParser.add(FOR_BEGIN, ctx.getStart());
     }
 
     @Override
-    public void exitFor_stmt(@NotNull Python3Parser.For_stmtContext ctx) {
+    public void exitFor_stmt(Python3Parser.For_stmtContext ctx) {
         jplagParser.addEnd(FOR_END, ctx.getStart());
     }
 
     @Override
-    public void enterDel_stmt(@NotNull Python3Parser.Del_stmtContext ctx) {
+    public void enterDel_stmt(Python3Parser.Del_stmtContext ctx) {
         jplagParser.add(DEL, ctx.getStart());
     }
 
     @Override
-    public void exitDel_stmt(@NotNull Python3Parser.Del_stmtContext ctx) {
+    public void exitDel_stmt(Python3Parser.Del_stmtContext ctx) {
     }
 
     @Override
-    public void enterAtom(@NotNull Python3Parser.AtomContext ctx) {
+    public void enterAtom(Python3Parser.AtomContext ctx) {
     }
 
     @Override
-    public void exitAtom(@NotNull Python3Parser.AtomContext ctx) {
+    public void exitAtom(Python3Parser.AtomContext ctx) {
     }
 
     @Override
-    public void enterStmt(@NotNull Python3Parser.StmtContext ctx) {
+    public void enterStmt(Python3Parser.StmtContext ctx) {
     }
 
     @Override
-    public void exitStmt(@NotNull Python3Parser.StmtContext ctx) {
+    public void exitStmt(Python3Parser.StmtContext ctx) {
     }
 
     @Override
-    public void enterEveryRule(@NotNull ParserRuleContext ctx) {
+    public void enterEveryRule(ParserRuleContext ctx) {
     }
 
     @Override
-    public void exitEveryRule(@NotNull ParserRuleContext ctx) {
+    public void exitEveryRule(ParserRuleContext ctx) {
     }
 
     @Override
-    public void visitTerminal(@NotNull TerminalNode node) {
+    public void visitTerminal(TerminalNode node) {
         if (node.getText().equals("=")) {
             jplagParser.add(ASSIGN, node.getSymbol());
         } else if (node.getText().equals("finally")) {
@@ -719,7 +718,7 @@ public class JplagPython3Listener implements Python3Listener, Python3TokenConsta
     }
 
     @Override
-    public void visitErrorNode(@NotNull ErrorNode node) {
+    public void visitErrorNode(ErrorNode node) {
     }
 
 	@Override