Add support for python3 (Issue #6)

remove duplicate dependency

jplag.frontend.python-3/pom.xml

@@ -0,0 +1,94 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<artifactId>python-3</artifactId>
+	<version>0.0.1-SNAPSHOT</version>
+
+	<repositories>
+		<repository>
+			<id>ipdNexus</id>
+			<name>Recaa Repository</name>
+			<url>http://bob.ipd.kit.edu/nexus/content/repositories/public/</url>
+		</repository>
+	</repositories>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.codehaus.mojo</groupId>
+				<artifactId>build-helper-maven-plugin</artifactId>
+				<version>1.7</version>
+				<configuration>
+					<sources>
+						<source>target/generated-sources/antlr4</source>
+					</sources>
+				</configuration>
+				<executions>
+					<execution>
+						<id>generate-antlr-sources</id>
+						<phase>generate-sources</phase>
+						<goals>
+							<goal>add-source</goal>
+						</goals>
+					</execution>
+					<execution>
+						<id>compile-with-antlr-sources</id>
+						<phase>compile</phase>
+						<goals>
+							<goal>add-source</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+			<plugin>
+				<groupId>org.antlr</groupId>
+				<artifactId>antlr4-maven-plugin</artifactId>
+				<version>4.2</version>
+				<configuration>
+					<!-- Comma separated list of grammar files or pattern grammar files
+						By default, grammar file(s) is in ${basedir}/src/main/antlr -->
+					<!-- <grammars>*.g</grammars> -->
+					<grammars>Python3.g4</grammars>
+				</configuration>
+				<executions>
+					<execution>
+						<goals>
+							<goal>antlr4</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-jar-plugin</artifactId>
+				<configuration>
+					<archive>
+						<manifest>
+							<mainClass>jplag.python3.Parser</mainClass>
+							<packageName>jplag.python3</packageName>
+						</manifest>
+					</archive>
+				</configuration>
+			</plugin>
+
+		</plugins>
+	</build>
+
+	<dependencies>
+		<dependency>
+			<groupId>edu.kit.ipd.jplag</groupId>
+			<artifactId>frontend-utils</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.antlr</groupId>
+			<artifactId>antlr4-runtime</artifactId>
+		</dependency>
+	</dependencies>
+	<parent>
+		<groupId>edu.kit.ipd.jplag</groupId>
+		<artifactId>parent</artifactId>
+		<version>0.3-SNAPSHOT</version>
+		<relativePath>../jplag.parent/pom.xml</relativePath>
+	</parent>
+</project>

jplag.frontend.python-3/src/main/antlr4/jplag/python3/grammar/Python3.g4

@@ -0,0 +1,1557 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2014 by Bart Kiers
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Project      : python3-parser; an ANTLR4 grammar for Python 3
+ *                https://github.com/bkiers/python3-parser
+ * Developed by : Bart Kiers, bart@big-o.nl
+ */
+grammar Python3;
+
+// All comments that start with "///" are copy-pasted from
+// The Python Language Reference: https://docs.python.org/3.3/reference/grammar.html
+
+tokens { INDENT, DEDENT }
+
+@lexer::members {
+
+  // A queue where extra tokens are pushed on (see the NEWLINE lexer rule).
+  private java.util.LinkedList<Token> tokens = new java.util.LinkedList<>();
+
+  // The stack that keeps track of the indentation level.
+  private java.util.Stack<Integer> indents = new java.util.Stack<>();
+
+  // The amount of opened braces, brackets and parenthesis.
+  private int opened = 0;
+
+  // The most recently produced token.
+  private Token lastToken = null;
+
+  @Override
+  public void emit(Token t) {
+    super.setToken(t);
+    tokens.offer(t);
+  }
+
+  @Override
+  public Token nextToken() {
+
+    // Check if the end-of-file is ahead and there are still some DEDENTS expected.
+    if (_input.LA(1) == EOF && !this.indents.isEmpty()) {
+
+      // Remove any trailing EOF tokens from our buffer.
+      for (int i = tokens.size() - 1; i >= 0; i--) {
+        if (tokens.get(i).getType() == EOF) {
+          tokens.remove(i);
+        }
+      }
+
+      // First emit an extra line break that serves as the end of the statement.
+      this.emit(commonToken(Python3Parser.NEWLINE, "\n"));
+
+      // Now emit as much DEDENT tokens as needed.
+      while (!indents.isEmpty()) {
+        this.emit(createDedent());
+        indents.pop();
+      }
+
+      // Put the EOF back on the token stream.
+      this.emit(commonToken(Python3Parser.EOF, "<EOF>"));
+    }
+
+    Token next = super.nextToken();
+
+    if (next.getChannel() == Token.DEFAULT_CHANNEL) {
+      // Keep track of the last token on the default channel.
+      this.lastToken = next;
+    }
+
+    return tokens.isEmpty() ? next : tokens.poll();
+  }
+
+  private Token createDedent() {
+    CommonToken dedent = commonToken(Python3Parser.DEDENT, "");
+    dedent.setLine(this.lastToken.getLine());
+    return dedent;
+  }
+
+  private CommonToken commonToken(int type, String text) {
+    int stop = this.getCharIndex() - 1;
+    int start = text.isEmpty() ? stop : stop - text.length() + 1;
+    return new CommonToken(this._tokenFactorySourcePair, type, DEFAULT_TOKEN_CHANNEL, start, stop);
+  }
+
+  // Calculates the indentation of the provided spaces, taking the
+  // following rules into account:
+  //
+  // "Tabs are replaced (from left to right) by one to eight spaces
+  //  such that the total number of characters up to and including
+  //  the replacement is a multiple of eight [...]"
+  //
+  //  -- https://docs.python.org/3.1/reference/lexical_analysis.html#indentation
+  static int getIndentationCount(String spaces) {
+
+    int count = 0;
+
+    for (char ch : spaces.toCharArray()) {
+      switch (ch) {
+        case '\t':
+          count += 8 - (count % 8);
+          break;
+        default:
+          // A normal space char.
+          count++;
+      }
+    }
+
+    return count;
+  }
+
+  boolean atStartOfInput() {
+    return super.getCharPositionInLine() == 0 && super.getLine() == 1;
+  }
+}
+
+/*
+ * parser rules
+ */
+
+/// single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+single_input
+ : NEWLINE
+ | simple_stmt
+ | compound_stmt NEWLINE
+ ;
+
+/// file_input: (NEWLINE | stmt)* ENDMARKER
+file_input
+ : ( NEWLINE | stmt )* EOF
+ ;
+
+/// eval_input: testlist NEWLINE* ENDMARKER
+eval_input
+ : testlist NEWLINE* EOF
+ ;
+
+/// decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+decorator
+ : '@' dotted_name ( '(' arglist? ')' )? NEWLINE
+ ;
+
+/// decorators: decorator+
+decorators
+ : decorator+
+ ;
+
+/// decorated: decorators (classdef | funcdef)
+decorated
+ : decorators ( classdef | funcdef )
+ ;
+
+/// funcdef: 'def' NAME parameters ['->' test] ':' suite
+funcdef
+ : DEF NAME parameters ( '->' test )? ':' suite
+ ;
+
+/// parameters: '(' [typedargslist] ')'
+parameters
+ : '(' typedargslist? ')'
+ ;
+
+/// typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
+///                ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
+///              |  '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
+typedargslist
+ : tfpdef ( '=' test )? ( ',' tfpdef ( '=' test )? )* ( ',' ( '*' tfpdef? ( ',' tfpdef ( '=' test )? )* ( ',' '**' tfpdef )? 
+                                                            | '**' tfpdef 
+                                                            )? 
+                                                      )?
+ | '*' tfpdef? ( ',' tfpdef ( '=' test )? )* ( ',' '**' tfpdef )? 
+ | '**' tfpdef
+ ;
+
+/// tfpdef: NAME [':' test]
+tfpdef
+ : NAME ( ':' test )?
+ ;
+
+/// varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
+///       ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]]
+///     |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
+varargslist
+ : vfpdef ( '=' test )? ( ',' vfpdef ( '=' test )? )* ( ',' ( '*' vfpdef? ( ',' vfpdef ( '=' test )? )* ( ',' '**' vfpdef )? 
+                                                            | '**' vfpdef 
+                                                            )? 
+                                                      )?
+ | '*' vfpdef? ( ',' vfpdef ( '=' test )? )* ( ',' '**' vfpdef )?
+ | '**' vfpdef
+ ;
+
+/// vfpdef: NAME
+vfpdef
+ : NAME
+ ;
+
+/// stmt: simple_stmt | compound_stmt
+stmt
+ : simple_stmt 
+ | compound_stmt
+ ;
+
+/// simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
+simple_stmt
+ : small_stmt ( ';' small_stmt )* ';'? NEWLINE
+ ;
+
+/// small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
+///              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
+small_stmt
+ : expr_stmt 
+ | del_stmt 
+ | pass_stmt 
+ | flow_stmt 
+ | import_stmt 
+ | global_stmt 
+ | nonlocal_stmt 
+ | assert_stmt
+ ;
+
+/// expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) |
+///                      ('=' (yield_expr|testlist_star_expr))*)
+expr_stmt
+ : testlist_star_expr ( augassign ( yield_expr | testlist) 
+                      | ( '=' ( yield_expr| testlist_star_expr ) )*
+                      )
+ ;           
+
+/// testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
+testlist_star_expr
+ : ( test | star_expr ) ( ',' ( test |  star_expr ) )* ','?
+ ;
+
+/// augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
+///             '<<=' | '>>=' | '**=' | '//=')
+augassign
+ : '+=' 
+ | '-=' 
+ | '*=' 
+ | '@=' // PEP 465
+ | '/=' 
+ | '%=' 
+ | '&=' 
+ | '|=' 
+ | '^=' 
+ | '<<=' 
+ | '>>=' 
+ | '**=' 
+ | '//='
+ ;
+
+/// del_stmt: 'del' exprlist
+del_stmt
+ : DEL exprlist
+ ;
+
+/// pass_stmt: 'pass'
+pass_stmt
+ : PASS
+ ;
+
+/// flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
+flow_stmt
+ : break_stmt 
+ | continue_stmt 
+ | return_stmt 
+ | raise_stmt 
+ | yield_stmt
+ ;
+
+/// break_stmt: 'break'
+break_stmt
+ : BREAK
+ ;
+
+/// continue_stmt: 'continue'
+continue_stmt
+ : CONTINUE
+ ;
+
+/// return_stmt: 'return' [testlist]
+return_stmt
+ : RETURN testlist?
+ ;
+
+/// yield_stmt: yield_expr
+yield_stmt
+ : yield_expr
+ ;
+
+/// raise_stmt: 'raise' [test ['from' test]]
+raise_stmt
+ : RAISE ( test ( FROM test )? )?
+ ;
+
+/// import_stmt: import_name | import_from
+import_stmt
+ : import_name 
+ | import_from
+ ;
+
+/// import_name: 'import' dotted_as_names
+import_name
+ : IMPORT dotted_as_names
+ ;
+
+/// # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
+/// import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
+///               'import' ('*' | '(' import_as_names ')' | import_as_names))
+import_from
+ : FROM ( ( '.' | '...' )* dotted_name 
+        | ('.' | '...')+ 
+        )
+   IMPORT ( '*' 
+          | '(' import_as_names ')' 
+          | import_as_names
+          )         
+ ;
+
+/// import_as_name: NAME ['as' NAME]
+import_as_name
+ : NAME ( AS NAME )?
+ ;
+
+/// dotted_as_name: dotted_name ['as' NAME]
+dotted_as_name
+ : dotted_name ( AS NAME )?
+ ;
+
+/// import_as_names: import_as_name (',' import_as_name)* [',']
+import_as_names
+ : import_as_name ( ',' import_as_name )* ','?
+ ;
+
+/// dotted_as_names: dotted_as_name (',' dotted_as_name)*
+dotted_as_names
+ : dotted_as_name ( ',' dotted_as_name )*
+ ;
+
+/// dotted_name: NAME ('.' NAME)*
+dotted_name
+ : NAME ( '.' NAME )*
+ ;
+
+/// global_stmt: 'global' NAME (',' NAME)*
+global_stmt
+ : GLOBAL NAME ( ',' NAME )*
+ ;
+
+/// nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
+nonlocal_stmt
+ : NONLOCAL NAME ( ',' NAME )*
+ ;
+
+/// assert_stmt: 'assert' test [',' test]
+assert_stmt
+ : ASSERT test ( ',' test )?
+ ;
+
+/// compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
+compound_stmt
+ : if_stmt 
+ | while_stmt 
+ | for_stmt 
+ | try_stmt 
+ | with_stmt 
+ | funcdef 
+ | classdef 
+ | decorated
+ ;
+
+/// if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+if_stmt
+ : IF test ':' suite ( ELIF test ':' suite )* ( ELSE ':' suite )?
+ ;
+
+/// while_stmt: 'while' test ':' suite ['else' ':' suite]
+while_stmt
+ : WHILE test ':' suite ( ELSE ':' suite )?
+ ;
+
+/// for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
+for_stmt
+ : FOR exprlist IN testlist ':' suite ( ELSE ':' suite )?
+ ;
+
+/// try_stmt: ('try' ':' suite
+///            ((except_clause ':' suite)+
+///       ['else' ':' suite]
+///       ['finally' ':' suite] |
+///      'finally' ':' suite))
+try_stmt
+ : TRY ':' suite ( ( except_clause ':' suite )+ 
+                   ( ELSE ':' suite )? 
+                   ( FINALLY ':' suite )?
+                 | FINALLY ':' suite
+                 )
+ ;
+
+/// with_stmt: 'with' with_item (',' with_item)*  ':' suite
+with_stmt
+ : WITH with_item ( ',' with_item )* ':' suite
+ ;
+
+/// with_item: test ['as' expr]
+with_item
+ : test ( AS expr )?
+ ;
+
+/// # NB compile.c makes sure that the default except clause is last
+/// except_clause: 'except' [test ['as' NAME]]
+except_clause
+ : EXCEPT ( test ( AS NAME )? )?
+ ;
+
+/// suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
+suite
+ : simple_stmt 
+ | NEWLINE INDENT stmt+ DEDENT
+ ;
+
+/// test: or_test ['if' or_test 'else' test] | lambdef
+test
+ : or_test ( IF or_test ELSE test )?
+ | lambdef
+ ;
+
+/// test_nocond: or_test | lambdef_nocond
+test_nocond
+ : or_test 
+ | lambdef_nocond
+ ;
+
+/// lambdef: 'lambda' [varargslist] ':' test
+lambdef
+ : LAMBDA varargslist? ':' test
+ ;
+
+/// lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
+lambdef_nocond
+ : LAMBDA varargslist? ':' test_nocond
+ ;
+
+/// or_test: and_test ('or' and_test)*
+or_test
+ : and_test ( OR and_test )*
+ ;
+
+/// and_test: not_test ('and' not_test)*
+and_test
+ : not_test ( AND not_test )*
+ ;
+
+/// not_test: 'not' not_test | comparison
+not_test
+ : NOT not_test 
+ | comparison
+ ;
+
+/// comparison: star_expr (comp_op star_expr)*
+comparison
+ : star_expr ( comp_op star_expr )*
+ ;
+
+/// # <> isn't actually a valid comparison operator in Python. It's here for the
+/// # sake of a __future__ import described in PEP 401
+/// comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+comp_op
+ : '<'
+ | '>'
+ | '=='
+ | '>='
+ | '<='
+ | '<>'
+ | '!='
+ | IN
+ | NOT IN
+ | IS
+ | IS NOT
+ ;
+
+/// star_expr: ['*'] expr
+star_expr
+ : '*'? expr
+ ;
+
+/// expr: xor_expr ('|' xor_expr)*
+expr
+ : xor_expr ( '|' xor_expr )*
+ ;
+
+/// xor_expr: and_expr ('^' and_expr)*
+xor_expr
+ : and_expr ( '^' and_expr )*
+ ;
+
+/// and_expr: shift_expr ('&' shift_expr)*
+and_expr
+ : shift_expr ( '&' shift_expr )*
+ ;
+
+/// shift_expr: arith_expr (('<<'|'>>') arith_expr)*
+shift_expr
+ : arith_expr ( '<<' arith_expr 
+              | '>>' arith_expr 
+              )*
+ ;
+
+/// arith_expr: term (('+'|'-') term)*
+arith_expr
+ : term ( '+' term
+        | '-' term 
+        )*
+ ;
+
+/// term: factor (('*'|'/'|'%'|'//') factor)*
+term
+ : factor ( '*' factor
+          | '/' factor
+          | '%' factor 
+          | '//' factor 
+          | '@' factor // PEP 465
+          )*
+ ;
+
+/// factor: ('+'|'-'|'~') factor | power
+factor
+ : '+' factor 
+ | '-' factor 
+ | '~' factor 
+ | power
+ ;
+
+/// power: atom trailer* ['**' factor]
+power
+ : atom trailer* ( '**' factor )?
+ ;
+
+/// atom: ('(' [yield_expr|testlist_comp] ')' |
+///        '[' [testlist_comp] ']' |
+///        '{' [dictorsetmaker] '}' |
+///        NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False')
+atom
+ : '(' ( yield_expr | testlist_comp )? ')' 
+ | '[' testlist_comp? ']'  
+ | '{' dictorsetmaker? '}' 
+ | NAME 
+ | number 
+ | string+ 
+ | '...' 
+ | NONE
+ | TRUE
+ | FALSE
+ ;
+
+/// testlist_comp: test ( comp_for | (',' test)* [','] )
+testlist_comp
+ : test ( comp_for 
+        | ( ',' test )* ','? 
+        )
+ ;
+
+/// trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
+trailer
+ : '(' arglist? ')' 
+ | '[' subscriptlist ']' 
+ | '.' NAME
+ ;
+
+/// subscriptlist: subscript (',' subscript)* [',']
+subscriptlist
+ : subscript ( ',' subscript )* ','?
+ ;
+
+/// subscript: test | [test] ':' [test] [sliceop]
+subscript
+ : test 
+ | test? ':' test? sliceop?
+ ;
+
+/// sliceop: ':' [test]
+sliceop
+ : ':' test?
+ ;
+
+/// exprlist: star_expr (',' star_expr)* [',']
+exprlist
+ : star_expr ( ',' star_expr )* ','?
+ ;
+
+/// testlist: test (',' test)* [',']
+testlist
+ : test ( ',' test )* ','?
+ ;
+
+/// dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
+///                   (test (comp_for | (',' test)* [','])) )
+dictorsetmaker
+ : test ':' test ( comp_for 
+                 | ( ',' test ':' test )* ','? 
+                 ) 
+ | test ( comp_for 
+        | ( ',' test )* ','? 
+        )
+ ;
+
+/// classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
+classdef
+ : CLASS NAME ( '(' arglist? ')' )? ':' suite
+ ;
+
+/// arglist: (argument ',')* (argument [',']
+///                          |'*' test (',' argument)* [',' '**' test]
+///                          |'**' test)
+arglist
+ : ( argument ',' )* ( argument ','?
+                     | '*' test ( ',' argument )* ( ',' '**' test )?
+                     | '**' test
+                     )
+ ;
+
+/// # The reason that keywords are test nodes instead of NAME is that using NAME
+/// # results in an ambiguity. ast.c makes sure it's a NAME.
+/// argument: test [comp_for] | test '=' test  # Really [keyword '='] test
+argument
+ : test comp_for? 
+ | test '=' test
+ ;
+
+/// comp_iter: comp_for | comp_if
+comp_iter
+ : comp_for 
+ | comp_if
+ ;
+
+/// comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_for
+ : FOR exprlist IN or_test comp_iter?
+ ;
+
+/// comp_if: 'if' test_nocond [comp_iter]
+comp_if
+ : IF test_nocond comp_iter?
+ ;
+
+/// yield_expr: 'yield' [testlist]
+yield_expr
+ : YIELD yield_arg?
+ ;
+
+/// yield_arg: 'from' test | testlist
+yield_arg
+ : FROM test 
+ | testlist
+ ;
+
+string
+ : STRING_LITERAL
+ | BYTES_LITERAL
+ ;
+
+number
+ : integer
+ | FLOAT_NUMBER
+ | IMAG_NUMBER
+ ;
+
+/// integer        ::=  decimalinteger | octinteger | hexinteger | bininteger
+integer
+ : DECIMAL_INTEGER
+ | OCT_INTEGER
+ | HEX_INTEGER
+ | BIN_INTEGER
+ ;
+
+/*
+ * lexer rules
+ */
+
+DEF : 'def';
+RETURN : 'return';
+RAISE : 'raise';
+FROM : 'from';
+IMPORT : 'import';
+AS : 'as';
+GLOBAL : 'global';
+NONLOCAL : 'nonlocal';
+ASSERT : 'assert';
+IF : 'if';
+ELIF : 'elif';
+ELSE : 'else';
+WHILE : 'while';
+FOR : 'for';
+IN : 'in';
+TRY : 'try';
+FINALLY : 'finally';
+WITH : 'with';
+EXCEPT : 'except';
+LAMBDA : 'lambda';
+OR : 'or';
+AND : 'and';
+NOT : 'not';
+IS : 'is';
+NONE : 'None';
+TRUE : 'True';
+FALSE : 'False';
+CLASS : 'class';
+YIELD : 'yield';
+DEL : 'del';
+PASS : 'pass';
+CONTINUE : 'continue';
+BREAK : 'break';
+
+NEWLINE
+ : ( {atStartOfInput()}?   SPACES
+   | ( '\r'? '\n' | '\r' ) SPACES?
+   )
+   {
+     String newLine = getText().replaceAll("[^\r\n]+", "");
+     String spaces = getText().replaceAll("[\r\n]+", "");
+     int next = _input.LA(1);
+
+     if (opened > 0 || next == '\r' || next == '\n' || next == '#') {
+       // If we're inside a list or on a blank line, ignore all indents, 
+       // dedents and line breaks.
+       skip();
+     }
+     else {
+       emit(commonToken(NEWLINE, newLine));
+
+       int indent = getIndentationCount(spaces);
+       int previous = indents.isEmpty() ? 0 : indents.peek();
+
+       if (indent == previous) {
+         // skip indents of the same size as the present indent-size
+         skip();
+       }
+       else if (indent > previous) {
+         indents.push(indent);
+         emit(commonToken(Python3Parser.INDENT, spaces));
+       }
+       else {
+         // Possibly emit more than 1 DEDENT token.
+         while(!indents.isEmpty() && indents.peek() > indent) {
+           this.emit(createDedent());
+           indents.pop();
+         }
+       }
+     }
+   }
+ ;
+
+/// identifier   ::=  id_start id_continue*
+NAME
+ : ID_START ID_CONTINUE*
+ ;
+
+/// stringliteral   ::=  [stringprefix](shortstring | longstring)
+/// stringprefix    ::=  "r" | "R"
+STRING_LITERAL
+ : [uU]? [rR]? ( SHORT_STRING | LONG_STRING )
+ ;
+
+/// bytesliteral   ::=  bytesprefix(shortbytes | longbytes)
+/// bytesprefix    ::=  "b" | "B" | "br" | "Br" | "bR" | "BR"
+BYTES_LITERAL
+ : [bB] [rR]? ( SHORT_BYTES | LONG_BYTES )
+ ;
+
+/// decimalinteger ::=  nonzerodigit digit* | "0"+
+DECIMAL_INTEGER
+ : NON_ZERO_DIGIT DIGIT*
+ | '0'+
+ ;
+
+/// octinteger     ::=  "0" ("o" | "O") octdigit+
+OCT_INTEGER
+ : '0' [oO] OCT_DIGIT+
+ ;
+
+/// hexinteger     ::=  "0" ("x" | "X") hexdigit+
+HEX_INTEGER
+ : '0' [xX] HEX_DIGIT+
+ ;
+
+/// bininteger     ::=  "0" ("b" | "B") bindigit+
+BIN_INTEGER
+ : '0' [bB] BIN_DIGIT+
+ ;
+
+/// floatnumber   ::=  pointfloat | exponentfloat
+FLOAT_NUMBER
+ : POINT_FLOAT
+ | EXPONENT_FLOAT
+ ;
+
+/// imagnumber ::=  (floatnumber | intpart) ("j" | "J")
+IMAG_NUMBER
+ : ( FLOAT_NUMBER | INT_PART ) [jJ]
+ ;
+
+DOT : '.';
+ELLIPSIS : '...';
+STAR : '*';
+OPEN_PAREN : '(' {opened++;};
+CLOSE_PAREN : ')' {opened--;};
+COMMA : ',';
+COLON : ':';
+SEMI_COLON : ';';
+POWER : '**';
+ASSIGN : '=';
+OPEN_BRACK : '[' {opened++;};
+CLOSE_BRACK : ']' {opened--;};
+OR_OP : '|';
+XOR : '^';
+AND_OP : '&';
+LEFT_SHIFT : '<<';
+RIGHT_SHIFT : '>>';
+ADD : '+';
+MINUS : '-';
+DIV : '/';
+MOD : '%';
+IDIV : '//';
+NOT_OP : '~';
+OPEN_BRACE : '{' {opened++;};
+CLOSE_BRACE : '}' {opened--;};
+LESS_THAN : '<';
+GREATER_THAN : '>';
+EQUALS : '==';
+GT_EQ : '>=';
+LT_EQ : '<=';
+NOT_EQ_1 : '<>';
+NOT_EQ_2 : '!=';
+AT : '@';
+ARROW : '->';
+ADD_ASSIGN : '+=';
+SUB_ASSIGN : '-=';
+MULT_ASSIGN : '*=';
+AT_ASSIGN : '@=';
+DIV_ASSIGN : '/=';
+MOD_ASSIGN : '%=';
+AND_ASSIGN : '&=';
+OR_ASSIGN : '|=';
+XOR_ASSIGN : '^=';
+LEFT_SHIFT_ASSIGN : '<<=';
+RIGHT_SHIFT_ASSIGN : '>>=';
+POWER_ASSIGN : '**=';
+IDIV_ASSIGN : '//=';
+
+SKIP_
+ : ( SPACES | COMMENT | LINE_JOINING ) -> skip
+ ;
+
+UNKNOWN_CHAR
+ : .
+ ;
+
+/* 
+ * fragments 
+ */
+
+/// shortstring     ::=  "'" shortstringitem* "'" | '"' shortstringitem* '"'
+/// shortstringitem ::=  shortstringchar | stringescapeseq
+/// shortstringchar ::=  <any source character except "\" or newline or the quote>
+fragment SHORT_STRING
+ : '\'' ( STRING_ESCAPE_SEQ | ~[\\\r\n'] )* '\''
+ | '"' ( STRING_ESCAPE_SEQ | ~[\\\r\n"] )* '"'
+ ;
+
+/// longstring      ::=  "'''" longstringitem* "'''" | '"""' longstringitem* '"""'
+fragment LONG_STRING
+ : '\'\'\'' LONG_STRING_ITEM*? '\'\'\''
+ | '"""' LONG_STRING_ITEM*? '"""'
+ ;
+
+/// longstringitem  ::=  longstringchar | stringescapeseq
+fragment LONG_STRING_ITEM
+ : LONG_STRING_CHAR
+ | STRING_ESCAPE_SEQ
+ ;
+
+/// longstringchar  ::=  <any source character except "\">
+fragment LONG_STRING_CHAR
+ : ~'\\'
+ ;
+
+/// stringescapeseq ::=  "\" <any source character>
+fragment STRING_ESCAPE_SEQ
+ : '\\' .
+ ;
+
+/// nonzerodigit   ::=  "1"..."9"
+fragment NON_ZERO_DIGIT
+ : [1-9]
+ ;
+
+/// digit          ::=  "0"..."9"
+fragment DIGIT
+ : [0-9]
+ ;
+
+/// octdigit       ::=  "0"..."7"
+fragment OCT_DIGIT
+ : [0-7]
+ ;
+
+/// hexdigit       ::=  digit | "a"..."f" | "A"..."F"
+fragment HEX_DIGIT
+ : [0-9a-fA-F]
+ ;
+
+/// bindigit       ::=  "0" | "1"
+fragment BIN_DIGIT
+ : [01]
+ ;
+
+/// pointfloat    ::=  [intpart] fraction | intpart "."
+fragment POINT_FLOAT
+ : INT_PART? FRACTION
+ | INT_PART '.'
+ ;
+
+/// exponentfloat ::=  (intpart | pointfloat) exponent
+fragment EXPONENT_FLOAT
+ : ( INT_PART | POINT_FLOAT ) EXPONENT
+ ;
+
+/// intpart       ::=  digit+
+fragment INT_PART
+ : DIGIT+
+ ;
+
+/// fraction      ::=  "." digit+
+fragment FRACTION
+ : '.' DIGIT+
+ ;
+
+/// exponent      ::=  ("e" | "E") ["+" | "-"] digit+
+fragment EXPONENT
+ : [eE] [+-]? DIGIT+
+ ;
+
+/// shortbytes     ::=  "'" shortbytesitem* "'" | '"' shortbytesitem* '"'
+/// shortbytesitem ::=  shortbyteschar | bytesescapeseq
+fragment SHORT_BYTES
+ : '\'' ( SHORT_BYTES_CHAR_NO_SINGLE_QUOTE | BYTES_ESCAPE_SEQ )* '\''
+ | '"' ( SHORT_BYTES_CHAR_NO_DOUBLE_QUOTE | BYTES_ESCAPE_SEQ )* '"'
+ ;
+    
+/// longbytes      ::=  "'''" longbytesitem* "'''" | '"""' longbytesitem* '"""'
+fragment LONG_BYTES
+ : '\'\'\'' LONG_BYTES_ITEM*? '\'\'\''
+ | '"""' LONG_BYTES_ITEM*? '"""'
+ ;
+
+/// longbytesitem  ::=  longbyteschar | bytesescapeseq
+fragment LONG_BYTES_ITEM
+ : LONG_BYTES_CHAR
+ | BYTES_ESCAPE_SEQ
+ ;
+
+/// shortbyteschar ::=  <any ASCII character except "\" or newline or the quote>
+fragment SHORT_BYTES_CHAR_NO_SINGLE_QUOTE
+ : [\u0000-\u0009]
+ | [\u000B-\u000C]
+ | [\u000E-\u0026]
+ | [\u0028-\u005B]
+ | [\u005D-\u007F]
+ ; 
+
+fragment SHORT_BYTES_CHAR_NO_DOUBLE_QUOTE
+ : [\u0000-\u0009]
+ | [\u000B-\u000C]
+ | [\u000E-\u0021]
+ | [\u0023-\u005B]
+ | [\u005D-\u007F]
+ ; 
+
+/// longbyteschar  ::=  <any ASCII character except "\">
+fragment LONG_BYTES_CHAR
+ : [\u0000-\u005B]
+ | [\u005D-\u007F]
+ ;
+
+/// bytesescapeseq ::=  "\" <any ASCII character>
+fragment BYTES_ESCAPE_SEQ
+ : '\\' [\u0000-\u007F]
+ ;
+
+fragment SPACES
+ : [ \t]+
+ ;
+
+fragment COMMENT
+ : '#' ~[\r\n]*
+ ;
+
+fragment LINE_JOINING
+ : '\\' SPACES? ( '\r'? '\n' | '\r' )
+ ;
+
+/// id_start     ::=  <all characters in general categories Lu, Ll, Lt, Lm, Lo, Nl, the underscore, and characters with the Other_ID_Start property>
+fragment ID_START
+ : '_'
+ | [A-Z]
+ | [a-z]
+ | '\u00AA'
+ | '\u00B5'
+ | '\u00BA'
+ | [\u00C0-\u00D6]
+ | [\u00D8-\u00F6]
+ | [\u00F8-\u01BA]
+ | '\u01BB'
+ | [\u01BC-\u01BF]
+ | [\u01C0-\u01C3]
+ | [\u01C4-\u0241]
+ | [\u0250-\u02AF]
+ | [\u02B0-\u02C1]
+ | [\u02C6-\u02D1]
+ | [\u02E0-\u02E4]
+ | '\u02EE'
+ | '\u037A'
+ | '\u0386'
+ | [\u0388-\u038A]
+ | '\u038C'
+ | [\u038E-\u03A1]
+ | [\u03A3-\u03CE]
+ | [\u03D0-\u03F5]
+ | [\u03F7-\u0481]
+ | [\u048A-\u04CE]
+ | [\u04D0-\u04F9]
+ | [\u0500-\u050F]
+ | [\u0531-\u0556]
+ | '\u0559'
+ | [\u0561-\u0587]
+ | [\u05D0-\u05EA]
+ | [\u05F0-\u05F2]
+ | [\u0621-\u063A]
+ | '\u0640'
+ | [\u0641-\u064A]
+ | [\u066E-\u066F]
+ | [\u0671-\u06D3]
+ | '\u06D5'
+ | [\u06E5-\u06E6]
+ | [\u06EE-\u06EF]
+ | [\u06FA-\u06FC]
+ | '\u06FF'
+ | '\u0710'
+ | [\u0712-\u072F]
+ | [\u074D-\u076D]
+ | [\u0780-\u07A5]
+ | '\u07B1'
+ | [\u0904-\u0939]
+ | '\u093D'
+ | '\u0950'
+ | [\u0958-\u0961]
+ | '\u097D'
+ | [\u0985-\u098C]
+ | [\u098F-\u0990]
+ | [\u0993-\u09A8]
+ | [\u09AA-\u09B0]
+ | '\u09B2'
+ | [\u09B6-\u09B9]
+ | '\u09BD'
+ | '\u09CE'
+ | [\u09DC-\u09DD]
+ | [\u09DF-\u09E1]
+ | [\u09F0-\u09F1]
+ | [\u0A05-\u0A0A]
+ | [\u0A0F-\u0A10]
+ | [\u0A13-\u0A28]
+ | [\u0A2A-\u0A30]
+ | [\u0A32-\u0A33]
+ | [\u0A35-\u0A36]
+ | [\u0A38-\u0A39]
+ | [\u0A59-\u0A5C]
+ | '\u0A5E'
+ | [\u0A72-\u0A74]
+ | [\u0A85-\u0A8D]
+ | [\u0A8F-\u0A91]
+ | [\u0A93-\u0AA8]
+ | [\u0AAA-\u0AB0]
+ | [\u0AB2-\u0AB3]
+ | [\u0AB5-\u0AB9]
+ | '\u0ABD'
+ | '\u0AD0'
+ | [\u0AE0-\u0AE1]
+ | [\u0B05-\u0B0C]
+ | [\u0B0F-\u0B10]
+ | [\u0B13-\u0B28]
+ | [\u0B2A-\u0B30]
+ | [\u0B32-\u0B33]
+ | [\u0B35-\u0B39]
+ | '\u0B3D'
+ | [\u0B5C-\u0B5D]
+ | [\u0B5F-\u0B61]
+ | '\u0B71'
+ | '\u0B83'
+ | [\u0B85-\u0B8A]
+ | [\u0B8E-\u0B90]
+ | [\u0B92-\u0B95]
+ | [\u0B99-\u0B9A]
+ | '\u0B9C'
+ | [\u0B9E-\u0B9F]
+ | [\u0BA3-\u0BA4]
+ | [\u0BA8-\u0BAA]
+ | [\u0BAE-\u0BB9]
+ | [\u0C05-\u0C0C]
+ | [\u0C0E-\u0C10]
+ | [\u0C12-\u0C28]
+ | [\u0C2A-\u0C33]
+ | [\u0C35-\u0C39]
+ | [\u0C60-\u0C61]
+ | [\u0C85-\u0C8C]
+ | [\u0C8E-\u0C90]
+ | [\u0C92-\u0CA8]
+ | [\u0CAA-\u0CB3]
+ | [\u0CB5-\u0CB9]
+ | '\u0CBD'
+ | '\u0CDE'
+ | [\u0CE0-\u0CE1]
+ | [\u0D05-\u0D0C]
+ | [\u0D0E-\u0D10]
+ | [\u0D12-\u0D28]
+ | [\u0D2A-\u0D39]
+ | [\u0D60-\u0D61]
+ | [\u0D85-\u0D96]
+ | [\u0D9A-\u0DB1]
+ | [\u0DB3-\u0DBB]
+ | '\u0DBD'
+ | [\u0DC0-\u0DC6]
+ | [\u0E01-\u0E30]
+ | [\u0E32-\u0E33]
+ | [\u0E40-\u0E45]
+ | '\u0E46'
+ | [\u0E81-\u0E82]
+ | '\u0E84'
+ | [\u0E87-\u0E88]
+ | '\u0E8A'
+ | '\u0E8D'
+ | [\u0E94-\u0E97]
+ | [\u0E99-\u0E9F]
+ | [\u0EA1-\u0EA3]
+ | '\u0EA5'
+ | '\u0EA7'
+ | [\u0EAA-\u0EAB]
+ | [\u0EAD-\u0EB0]
+ | [\u0EB2-\u0EB3]
+ | '\u0EBD'
+ | [\u0EC0-\u0EC4]
+ | '\u0EC6'
+ | [\u0EDC-\u0EDD]
+ | '\u0F00'
+ | [\u0F40-\u0F47]
+ | [\u0F49-\u0F6A]
+ | [\u0F88-\u0F8B]
+ | [\u1000-\u1021]
+ | [\u1023-\u1027]
+ | [\u1029-\u102A]
+ | [\u1050-\u1055]
+ | [\u10A0-\u10C5]
+ | [\u10D0-\u10FA]
+ | '\u10FC'
+ | [\u1100-\u1159]
+ | [\u115F-\u11A2]
+ | [\u11A8-\u11F9]
+ | [\u1200-\u1248]
+ | [\u124A-\u124D]
+ | [\u1250-\u1256]
+ | '\u1258'
+ | [\u125A-\u125D]
+ | [\u1260-\u1288]
+ | [\u128A-\u128D]
+ | [\u1290-\u12B0]
+ | [\u12B2-\u12B5]
+ | [\u12B8-\u12BE]
+ | '\u12C0'
+ | [\u12C2-\u12C5]
+ | [\u12C8-\u12D6]
+ | [\u12D8-\u1310]
+ | [\u1312-\u1315]
+ | [\u1318-\u135A]
+ | [\u1380-\u138F]
+ | [\u13A0-\u13F4]
+ | [\u1401-\u166C]
+ | [\u166F-\u1676]
+ | [\u1681-\u169A]
+ | [\u16A0-\u16EA]
+ | [\u16EE-\u16F0]
+ | [\u1700-\u170C]
+ | [\u170E-\u1711]
+ | [\u1720-\u1731]
+ | [\u1740-\u1751]
+ | [\u1760-\u176C]
+ | [\u176E-\u1770]
+ | [\u1780-\u17B3]
+ | '\u17D7'
+ | '\u17DC'
+ | [\u1820-\u1842]
+ | '\u1843'
+ | [\u1844-\u1877]
+ | [\u1880-\u18A8]
+ | [\u1900-\u191C]
+ | [\u1950-\u196D]
+ | [\u1970-\u1974]
+ | [\u1980-\u19A9]
+ | [\u19C1-\u19C7]
+ | [\u1A00-\u1A16]
+ | [\u1D00-\u1D2B]
+ | [\u1D2C-\u1D61]
+ | [\u1D62-\u1D77]
+ | '\u1D78'
+ | [\u1D79-\u1D9A]
+ | [\u1D9B-\u1DBF]
+ | [\u1E00-\u1E9B]
+ | [\u1EA0-\u1EF9]
+ | [\u1F00-\u1F15]
+ | [\u1F18-\u1F1D]
+ | [\u1F20-\u1F45]
+ | [\u1F48-\u1F4D]
+ | [\u1F50-\u1F57]
+ | '\u1F59'
+ | '\u1F5B'
+ | '\u1F5D'
+ | [\u1F5F-\u1F7D]
+ | [\u1F80-\u1FB4]
+ | [\u1FB6-\u1FBC]
+ | '\u1FBE'
+ | [\u1FC2-\u1FC4]
+ | [\u1FC6-\u1FCC]
+ | [\u1FD0-\u1FD3]
+ | [\u1FD6-\u1FDB]
+ | [\u1FE0-\u1FEC]
+ | [\u1FF2-\u1FF4]
+ | [\u1FF6-\u1FFC]
+ | '\u2071'
+ | '\u207F'
+ | [\u2090-\u2094]
+ | '\u2102'
+ | '\u2107'
+ | [\u210A-\u2113]
+ | '\u2115'
+ | '\u2118'
+ | [\u2119-\u211D]
+ | '\u2124'
+ | '\u2126'
+ | '\u2128'
+ | [\u212A-\u212D]
+ | '\u212E'
+ | [\u212F-\u2131]
+ | [\u2133-\u2134]
+ | [\u2135-\u2138]
+ | '\u2139'
+ | [\u213C-\u213F]
+ | [\u2145-\u2149]
+ | [\u2160-\u2183]
+ | [\u2C00-\u2C2E]
+ | [\u2C30-\u2C5E]
+ | [\u2C80-\u2CE4]
+ | [\u2D00-\u2D25]
+ | [\u2D30-\u2D65]
+ | '\u2D6F'
+ | [\u2D80-\u2D96]
+ | [\u2DA0-\u2DA6]
+ | [\u2DA8-\u2DAE]
+ | [\u2DB0-\u2DB6]
+ | [\u2DB8-\u2DBE]
+ | [\u2DC0-\u2DC6]
+ | [\u2DC8-\u2DCE]
+ | [\u2DD0-\u2DD6]
+ | [\u2DD8-\u2DDE]
+ | '\u3005'
+ | '\u3006'
+ | '\u3007'
+ | [\u3021-\u3029]
+ | [\u3031-\u3035]
+ | [\u3038-\u303A]
+ | '\u303B'
+ | '\u303C'
+ | [\u3041-\u3096]
+ | [\u309B-\u309C]
+ | [\u309D-\u309E]
+ | '\u309F'
+ | [\u30A1-\u30FA]
+ | [\u30FC-\u30FE]
+ | '\u30FF'
+ | [\u3105-\u312C]
+ | [\u3131-\u318E]
+ | [\u31A0-\u31B7]
+ | [\u31F0-\u31FF]
+ | [\u3400-\u4DB5]
+ | [\u4E00-\u9FBB]
+ | [\uA000-\uA014]
+ | '\uA015'
+ | [\uA016-\uA48C]
+ | [\uA800-\uA801]
+ | [\uA803-\uA805]
+ | [\uA807-\uA80A]
+ | [\uA80C-\uA822]
+ | [\uAC00-\uD7A3]
+ | [\uF900-\uFA2D]
+ | [\uFA30-\uFA6A]
+ | [\uFA70-\uFAD9]
+ | [\uFB00-\uFB06]
+ | [\uFB13-\uFB17]
+ | '\uFB1D'
+ | [\uFB1F-\uFB28]
+ | [\uFB2A-\uFB36]
+ | [\uFB38-\uFB3C]
+ | '\uFB3E'
+ | [\uFB40-\uFB41]
+ | [\uFB43-\uFB44]
+ | [\uFB46-\uFBB1]
+ | [\uFBD3-\uFD3D]
+ | [\uFD50-\uFD8F]
+ | [\uFD92-\uFDC7]
+ | [\uFDF0-\uFDFB]
+ | [\uFE70-\uFE74]
+ | [\uFE76-\uFEFC]
+ | [\uFF21-\uFF3A]
+ | [\uFF41-\uFF5A]
+ | [\uFF66-\uFF6F]
+ | '\uFF70'
+ | [\uFF71-\uFF9D]
+ | [\uFF9E-\uFF9F]
+ | [\uFFA0-\uFFBE]
+ | [\uFFC2-\uFFC7]
+ | [\uFFCA-\uFFCF]
+ | [\uFFD2-\uFFD7]
+ | [\uFFDA-\uFFDC]
+ ;
+
+/// id_continue  ::=  <all characters in id_start, plus characters in the categories Mn, Mc, Nd, Pc and others with the Other_ID_Continue property>
+fragment ID_CONTINUE
+ : ID_START
+ | [0-9]
+ | [\u0300-\u036F]
+ | [\u0483-\u0486]
+ | [\u0591-\u05B9]
+ | [\u05BB-\u05BD]
+ | '\u05BF'
+ | [\u05C1-\u05C2]
+ | [\u05C4-\u05C5]
+ | '\u05C7'
+ | [\u0610-\u0615]
+ | [\u064B-\u065E]
+ | [\u0660-\u0669]
+ | '\u0670'
+ | [\u06D6-\u06DC]
+ | [\u06DF-\u06E4]
+ | [\u06E7-\u06E8]
+ | [\u06EA-\u06ED]
+ | [\u06F0-\u06F9]
+ | '\u0711'
+ | [\u0730-\u074A]
+ | [\u07A6-\u07B0]
+ | [\u0901-\u0902]
+ | '\u0903'
+ | '\u093C'
+ | [\u093E-\u0940]
+ | [\u0941-\u0948]
+ | [\u0949-\u094C]
+ | '\u094D'
+ | [\u0951-\u0954]
+ | [\u0962-\u0963]
+ | [\u0966-\u096F]
+ | '\u0981'
+ | [\u0982-\u0983]
+ | '\u09BC'
+ | [\u09BE-\u09C0]
+ | [\u09C1-\u09C4]
+ | [\u09C7-\u09C8]
+ | [\u09CB-\u09CC]
+ | '\u09CD'
+ | '\u09D7'
+ | [\u09E2-\u09E3]
+ | [\u09E6-\u09EF]
+ | [\u0A01-\u0A02]
+ | '\u0A03'
+ | '\u0A3C'
+ | [\u0A3E-\u0A40]
+ | [\u0A41-\u0A42]
+ | [\u0A47-\u0A48]
+ | [\u0A4B-\u0A4D]
+ | [\u0A66-\u0A6F]
+ | [\u0A70-\u0A71]
+ | [\u0A81-\u0A82]
+ | '\u0A83'
+ | '\u0ABC'
+ | [\u0ABE-\u0AC0]
+ | [\u0AC1-\u0AC5]
+ | [\u0AC7-\u0AC8]
+ | '\u0AC9'
+ | [\u0ACB-\u0ACC]
+ | '\u0ACD'
+ | [\u0AE2-\u0AE3]
+ | [\u0AE6-\u0AEF]
+ | '\u0B01'
+ | [\u0B02-\u0B03]
+ | '\u0B3C'
+ | '\u0B3E'
+ | '\u0B3F'
+ | '\u0B40'
+ | [\u0B41-\u0B43]
+ | [\u0B47-\u0B48]
+ | [\u0B4B-\u0B4C]
+ | '\u0B4D'
+ | '\u0B56'
+ | '\u0B57'
+ | [\u0B66-\u0B6F]
+ | '\u0B82'
+ | [\u0BBE-\u0BBF]
+ | '\u0BC0'
+ | [\u0BC1-\u0BC2]
+ | [\u0BC6-\u0BC8]
+ | [\u0BCA-\u0BCC]
+ | '\u0BCD'
+ | '\u0BD7'
+ | [\u0BE6-\u0BEF]
+ | [\u0C01-\u0C03]
+ | [\u0C3E-\u0C40]
+ | [\u0C41-\u0C44]
+ | [\u0C46-\u0C48]
+ | [\u0C4A-\u0C4D]
+ | [\u0C55-\u0C56]
+ | [\u0C66-\u0C6F]
+ | [\u0C82-\u0C83]
+ | '\u0CBC'
+ | '\u0CBE'
+ | '\u0CBF'
+ | [\u0CC0-\u0CC4]
+ | '\u0CC6'
+ | [\u0CC7-\u0CC8]
+ | [\u0CCA-\u0CCB]
+ | [\u0CCC-\u0CCD]
+ | [\u0CD5-\u0CD6]
+ | [\u0CE6-\u0CEF]
+ | [\u0D02-\u0D03]
+ | [\u0D3E-\u0D40]
+ | [\u0D41-\u0D43]
+ | [\u0D46-\u0D48]
+ | [\u0D4A-\u0D4C]
+ | '\u0D4D'
+ | '\u0D57'
+ | [\u0D66-\u0D6F]
+ | [\u0D82-\u0D83]
+ | '\u0DCA'
+ | [\u0DCF-\u0DD1]
+ | [\u0DD2-\u0DD4]
+ | '\u0DD6'
+ | [\u0DD8-\u0DDF]
+ | [\u0DF2-\u0DF3]
+ | '\u0E31'
+ | [\u0E34-\u0E3A]
+ | [\u0E47-\u0E4E]
+ | [\u0E50-\u0E59]
+ | '\u0EB1'
+ | [\u0EB4-\u0EB9]
+ | [\u0EBB-\u0EBC]
+ | [\u0EC8-\u0ECD]
+ | [\u0ED0-\u0ED9]
+ | [\u0F18-\u0F19]
+ | [\u0F20-\u0F29]
+ | '\u0F35'
+ | '\u0F37'
+ | '\u0F39'
+ | [\u0F3E-\u0F3F]
+ | [\u0F71-\u0F7E]
+ | '\u0F7F'
+ | [\u0F80-\u0F84]
+ | [\u0F86-\u0F87]
+ | [\u0F90-\u0F97]
+ | [\u0F99-\u0FBC]
+ | '\u0FC6'
+ | '\u102C'
+ | [\u102D-\u1030]
+ | '\u1031'
+ | '\u1032'
+ | [\u1036-\u1037]
+ | '\u1038'
+ | '\u1039'
+ | [\u1040-\u1049]
+ | [\u1056-\u1057]
+ | [\u1058-\u1059]
+ | '\u135F'
+ | [\u1369-\u1371]
+ | [\u1712-\u1714]
+ | [\u1732-\u1734]
+ | [\u1752-\u1753]
+ | [\u1772-\u1773]
+ | '\u17B6'
+ | [\u17B7-\u17BD]
+ | [\u17BE-\u17C5]
+ | '\u17C6'
+ | [\u17C7-\u17C8]
+ | [\u17C9-\u17D3]
+ | '\u17DD'
+ | [\u17E0-\u17E9]
+ | [\u180B-\u180D]
+ | [\u1810-\u1819]
+ | '\u18A9'
+ | [\u1920-\u1922]
+ | [\u1923-\u1926]
+ | [\u1927-\u1928]
+ | [\u1929-\u192B]
+ | [\u1930-\u1931]
+ | '\u1932'
+ | [\u1933-\u1938]
+ | [\u1939-\u193B]
+ | [\u1946-\u194F]
+ | [\u19B0-\u19C0]
+ | [\u19C8-\u19C9]
+ | [\u19D0-\u19D9]
+ | [\u1A17-\u1A18]
+ | [\u1A19-\u1A1B]
+ | [\u1DC0-\u1DC3]
+ | [\u203F-\u2040]
+ | '\u2054'
+ | [\u20D0-\u20DC]
+ | '\u20E1'
+ | [\u20E5-\u20EB]
+ | [\u302A-\u302F]
+ | [\u3099-\u309A]
+ | '\uA802'
+ | '\uA806'
+ | '\uA80B'
+ | [\uA823-\uA824]
+ | [\uA825-\uA826]
+ | '\uA827'
+ | '\uFB1E'
+ | [\uFE00-\uFE0F]
+ | [\uFE20-\uFE23]
+ | [\uFE33-\uFE34]
+ | [\uFE4D-\uFE4F]
+ | [\uFF10-\uFF19]
+ | '\uFF3F'
+ ;

jplag.frontend.python-3/src/main/java/jplag/python3/JplagPython3Listener.java

@@ -0,0 +1,740 @@
+package jplag.python3;
+
+import jplag.python3.grammar.*;
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.misc.NotNull;
+import org.antlr.v4.runtime.tree.ErrorNode;
+import org.antlr.v4.runtime.tree.TerminalNode;
+
+public class JplagPython3Listener implements Python3Listener, Python3TokenConstants {
+
+    private jplag.python3.Parser jplagParser;
+
+    public JplagPython3Listener(jplag.python3.Parser jplag) {
+        jplagParser = jplag;
+    }
+
+    @Override
+    public void enterTestlist(@NotNull Python3Parser.TestlistContext ctx) {
+    }
+
+    @Override
+    public void exitTestlist(@NotNull Python3Parser.TestlistContext ctx) {
+    }
+
+    @Override
+    public void enterAssert_stmt(@NotNull Python3Parser.Assert_stmtContext ctx) {
+        jplagParser.add(ASSERT, ctx.getStart());
+    }
+
+    @Override
+    public void exitAssert_stmt(@NotNull Python3Parser.Assert_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterArgument(@NotNull Python3Parser.ArgumentContext ctx) {
+    }
+
+    @Override
+    public void exitArgument(@NotNull Python3Parser.ArgumentContext ctx) {
+    }
+
+    @Override
+    public void enterNot_test(@NotNull Python3Parser.Not_testContext ctx) {
+    }
+
+    @Override
+    public void exitNot_test(@NotNull Python3Parser.Not_testContext ctx) {
+    }
+
+    @Override
+    public void enterFile_input(@NotNull Python3Parser.File_inputContext ctx) {
+    }
+
+    @Override
+    public void exitFile_input(@NotNull Python3Parser.File_inputContext ctx) {
+    }
+
+    @Override
+    public void enterXor_expr(@NotNull Python3Parser.Xor_exprContext ctx) {
+    }
+
+    @Override
+    public void exitXor_expr(@NotNull Python3Parser.Xor_exprContext ctx) {
+    }
+
+    @Override
+    public void enterInteger(@NotNull Python3Parser.IntegerContext ctx) {
+    }
+
+    @Override
+    public void exitInteger(@NotNull Python3Parser.IntegerContext ctx) {
+    }
+
+    @Override
+    public void enterImport_from(@NotNull Python3Parser.Import_fromContext ctx) {
+    }
+
+    @Override
+    public void exitImport_from(@NotNull Python3Parser.Import_fromContext ctx) {
+    }
+
+    @Override
+    public void enterSingle_input(@NotNull Python3Parser.Single_inputContext ctx) {
+    }
+
+    @Override
+    public void exitSingle_input(@NotNull Python3Parser.Single_inputContext ctx) {
+    }
+
+    @Override
+    public void enterDecorated(@NotNull Python3Parser.DecoratedContext ctx) {
+        jplagParser.add(DEC_BEGIN, ctx.getStart());
+    }
+
+    @Override
+    public void exitDecorated(@NotNull Python3Parser.DecoratedContext ctx) {
+        jplagParser.add(DEC_END, ctx.getStart());
+    }
+
+    @Override
+    public void enterWith_item(@NotNull Python3Parser.With_itemContext ctx) {
+    }
+
+    @Override
+    public void exitWith_item(@NotNull Python3Parser.With_itemContext ctx) {
+    }
+
+    @Override
+    public void enterRaise_stmt(@NotNull Python3Parser.Raise_stmtContext ctx) {
+        jplagParser.add(RAISE, ctx.getStart());
+    }
+
+    @Override
+    public void exitRaise_stmt(@NotNull Python3Parser.Raise_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterImport_as_name(@NotNull Python3Parser.Import_as_nameContext ctx) {
+    }
+
+    @Override
+    public void exitImport_as_name(@NotNull Python3Parser.Import_as_nameContext ctx) {
+    }
+
+    @Override
+    public void enterExcept_clause(@NotNull Python3Parser.Except_clauseContext ctx) {
+        jplagParser.add(EXCEPT_BEGIN, ctx.getStart());
+    }
+
+    @Override
+    public void exitExcept_clause(@NotNull Python3Parser.Except_clauseContext ctx) {
+        jplagParser.add(EXCEPT_END, ctx.getStart());
+    }
+
+    @Override
+    public void enterCompound_stmt(@NotNull Python3Parser.Compound_stmtContext ctx) {
+    }
+
+    @Override
+    public void exitCompound_stmt(@NotNull Python3Parser.Compound_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterNumber(@NotNull Python3Parser.NumberContext ctx) {
+    }
+
+    @Override
+    public void exitNumber(@NotNull Python3Parser.NumberContext ctx) {
+    }
+
+    @Override
+    public void enterAnd_expr(@NotNull Python3Parser.And_exprContext ctx) {
+    }
+
+    @Override
+    public void exitAnd_expr(@NotNull Python3Parser.And_exprContext ctx) {
+    }
+
+    @Override
+    public void enterLambdef_nocond(@NotNull Python3Parser.Lambdef_nocondContext ctx) {
+    }
+
+    @Override
+    public void exitLambdef_nocond(@NotNull Python3Parser.Lambdef_nocondContext ctx) {
+    }
+
+    @Override
+    public void enterDictorsetmaker(@NotNull Python3Parser.DictorsetmakerContext ctx) {
+        jplagParser.add(ARRAY, ctx.getStart());
+    }
+
+    @Override
+    public void exitDictorsetmaker(@NotNull Python3Parser.DictorsetmakerContext ctx) {
+    }
+
+    @Override
+    public void enterReturn_stmt(@NotNull Python3Parser.Return_stmtContext ctx) {
+        jplagParser.add(RETURN, ctx.getStart());
+    }
+
+    @Override
+    public void exitReturn_stmt(@NotNull Python3Parser.Return_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterDotted_name(@NotNull Python3Parser.Dotted_nameContext ctx) {
+    }
+
+    @Override
+    public void exitDotted_name(@NotNull Python3Parser.Dotted_nameContext ctx) {
+    }
+
+    @Override
+    public void enterFlow_stmt(@NotNull Python3Parser.Flow_stmtContext ctx) {
+    }
+
+    @Override
+    public void exitFlow_stmt(@NotNull Python3Parser.Flow_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterWhile_stmt(@NotNull Python3Parser.While_stmtContext ctx) {
+        jplagParser.add(WHILE_BEGIN, ctx.getStart());
+    }
+
+    @Override
+    public void exitWhile_stmt(@NotNull Python3Parser.While_stmtContext ctx) {
+        jplagParser.add(WHILE_END, ctx.getStart());
+    }
+
+    @Override
+    public void enterOr_test(@NotNull Python3Parser.Or_testContext ctx) {
+    }
+
+    @Override
+    public void exitOr_test(@NotNull Python3Parser.Or_testContext ctx) {
+    }
+
+    @Override
+    public void enterComparison(@NotNull Python3Parser.ComparisonContext ctx) {
+    }
+
+    @Override
+    public void exitComparison(@NotNull Python3Parser.ComparisonContext ctx) {
+    }
+
+    @Override
+    public void enterTest(@NotNull Python3Parser.TestContext ctx) {
+    }
+
+    @Override
+    public void exitTest(@NotNull Python3Parser.TestContext ctx) {
+    }
+
+    @Override
+    public void enterSubscript(@NotNull Python3Parser.SubscriptContext ctx) {
+    }
+
+    @Override
+    public void exitSubscript(@NotNull Python3Parser.SubscriptContext ctx) {
+    }
+
+    @Override
+    public void enterComp_for(@NotNull Python3Parser.Comp_forContext ctx) {
+    }
+
+    @Override
+    public void exitComp_for(@NotNull Python3Parser.Comp_forContext ctx) {
+    }
+
+    @Override
+    public void enterYield_arg(@NotNull Python3Parser.Yield_argContext ctx) {
+        jplagParser.add(YIELD, ctx.getStart());
+    }
+
+    @Override
+    public void exitYield_arg(@NotNull Python3Parser.Yield_argContext ctx) {
+    }
+
+    @Override
+    public void enterYield_expr(@NotNull Python3Parser.Yield_exprContext ctx) {
+    }
+
+    @Override
+    public void exitYield_expr(@NotNull Python3Parser.Yield_exprContext ctx) {
+    }
+
+    @Override
+    public void enterImport_stmt(@NotNull Python3Parser.Import_stmtContext ctx) {
+        jplagParser.add(IMPORT, ctx.getStart());
+    }
+
+    @Override
+    public void exitImport_stmt(@NotNull Python3Parser.Import_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterShift_expr(@NotNull Python3Parser.Shift_exprContext ctx) {
+    }
+
+    @Override
+    public void exitShift_expr(@NotNull Python3Parser.Shift_exprContext ctx) {
+    }
+
+    @Override
+    public void enterLambdef(@NotNull Python3Parser.LambdefContext ctx) {
+        jplagParser.add(LAMBDA, ctx.getStart());
+    }
+
+    @Override
+    public void exitLambdef(@NotNull Python3Parser.LambdefContext ctx) {
+    }
+
+    @Override
+    public void enterAnd_test(@NotNull Python3Parser.And_testContext ctx) {
+    }
+
+    @Override
+    public void exitAnd_test(@NotNull Python3Parser.And_testContext ctx) {
+    }
+
+    @Override
+    public void enterGlobal_stmt(@NotNull Python3Parser.Global_stmtContext ctx) {
+    }
+
+    @Override
+    public void exitGlobal_stmt(@NotNull Python3Parser.Global_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterImport_as_names(@NotNull Python3Parser.Import_as_namesContext ctx) {
+    }
+
+    @Override
+    public void exitImport_as_names(@NotNull Python3Parser.Import_as_namesContext ctx) {
+    }
+
+    @Override
+    public void enterDecorators(@NotNull Python3Parser.DecoratorsContext ctx) {
+    }
+
+    @Override
+    public void exitDecorators(@NotNull Python3Parser.DecoratorsContext ctx) {
+    }
+
+    @Override
+    public void enterTry_stmt(@NotNull Python3Parser.Try_stmtContext ctx) {
+        jplagParser.add(TRY_BEGIN, ctx.getStart());
+    }
+
+    @Override
+    public void exitTry_stmt(@NotNull Python3Parser.Try_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterComp_op(@NotNull Python3Parser.Comp_opContext ctx) {
+    }
+
+    @Override
+    public void exitComp_op(@NotNull Python3Parser.Comp_opContext ctx) {
+    }
+
+    @Override
+    public void enterStar_expr(@NotNull Python3Parser.Star_exprContext ctx) {
+    }
+
+    @Override
+    public void exitStar_expr(@NotNull Python3Parser.Star_exprContext ctx) {
+    }
+
+    @Override
+    public void enterBreak_stmt(@NotNull Python3Parser.Break_stmtContext ctx) {
+        jplagParser.add(BREAK, ctx.getStart());
+    }
+
+    @Override
+    public void exitBreak_stmt(@NotNull Python3Parser.Break_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterParameters(@NotNull Python3Parser.ParametersContext ctx) {
+    }
+
+    @Override
+    public void exitParameters(@NotNull Python3Parser.ParametersContext ctx) {
+    }
+
+    @Override
+    public void enterDecorator(@NotNull Python3Parser.DecoratorContext ctx) {
+    }
+
+    @Override
+    public void exitDecorator(@NotNull Python3Parser.DecoratorContext ctx) {
+    }
+
+    @Override
+    public void enterTfpdef(@NotNull Python3Parser.TfpdefContext ctx) {
+    }
+
+    @Override
+    public void exitTfpdef(@NotNull Python3Parser.TfpdefContext ctx) {
+    }
+
+    @Override
+    public void enterString(@NotNull Python3Parser.StringContext ctx) {
+    }
+
+    @Override
+    public void exitString(@NotNull Python3Parser.StringContext ctx) {
+    }
+
+    @Override
+    public void enterTestlist_comp(@NotNull Python3Parser.Testlist_compContext ctx) {
+        jplagParser.add(ARRAY, ctx.getStart());
+    }
+
+    @Override
+    public void exitTestlist_comp(@NotNull Python3Parser.Testlist_compContext ctx) {
+    }
+
+    @Override
+    public void enterIf_stmt(@NotNull Python3Parser.If_stmtContext ctx) {
+        jplagParser.add(IF_BEGIN, ctx.getStart());
+    }
+
+    @Override
+    public void exitIf_stmt(@NotNull Python3Parser.If_stmtContext ctx) {
+        jplagParser.add(IF_END, ctx.getStart());
+    }
+
+    @Override
+    public void enterWith_stmt(@NotNull Python3Parser.With_stmtContext ctx) {
+        jplagParser.add(WITH_BEGIN, ctx.getStart());
+    }
+
+    @Override
+    public void exitWith_stmt(@NotNull Python3Parser.With_stmtContext ctx) {
+        jplagParser.add(WITH_END, ctx.getStart());
+    }
+
+    @Override
+    public void enterClassdef(@NotNull Python3Parser.ClassdefContext ctx) {
+        jplagParser.add(CLASS_BEGIN, ctx.getStart());
+    }
+
+    @Override
+    public void exitClassdef(@NotNull Python3Parser.ClassdefContext ctx) {
+        jplagParser.add(CLASS_END, ctx.getStart());
+    }
+
+    @Override
+    public void enterExprlist(@NotNull Python3Parser.ExprlistContext ctx) {
+    }
+
+    @Override
+    public void exitExprlist(@NotNull Python3Parser.ExprlistContext ctx) {
+    }
+
+    @Override
+    public void enterSmall_stmt(@NotNull Python3Parser.Small_stmtContext ctx) {
+    }
+
+    @Override
+    public void exitSmall_stmt(@NotNull Python3Parser.Small_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterTrailer(@NotNull Python3Parser.TrailerContext ctx) {
+        jplagParser.add(APPLY, ctx.getStart());
+    }
+
+    @Override
+    public void exitTrailer(@NotNull Python3Parser.TrailerContext ctx) {
+    }
+
+    @Override
+    public void enterDotted_as_names(@NotNull Python3Parser.Dotted_as_namesContext ctx) {
+    }
+
+    @Override
+    public void exitDotted_as_names(@NotNull Python3Parser.Dotted_as_namesContext ctx) {
+    }
+
+    @Override
+    public void enterArith_expr(@NotNull Python3Parser.Arith_exprContext ctx) {
+    }
+
+    @Override
+    public void exitArith_expr(@NotNull Python3Parser.Arith_exprContext ctx) {
+    }
+
+    @Override
+    public void enterArglist(@NotNull Python3Parser.ArglistContext ctx) {
+    }
+
+    @Override
+    public void exitArglist(@NotNull Python3Parser.ArglistContext ctx) {
+    }
+
+    @Override
+    public void enterSimple_stmt(@NotNull Python3Parser.Simple_stmtContext ctx) {
+    }
+
+    @Override
+    public void exitSimple_stmt(@NotNull Python3Parser.Simple_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterTypedargslist(@NotNull Python3Parser.TypedargslistContext ctx) {
+    }
+
+    @Override
+    public void exitTypedargslist(@NotNull Python3Parser.TypedargslistContext ctx) {
+    }
+
+    @Override
+    public void enterExpr(@NotNull Python3Parser.ExprContext ctx) {
+    }
+
+    @Override
+    public void exitExpr(@NotNull Python3Parser.ExprContext ctx) {
+    }
+
+    @Override
+    public void enterTerm(@NotNull Python3Parser.TermContext ctx) {
+    }
+
+    @Override
+    public void exitTerm(@NotNull Python3Parser.TermContext ctx) {
+    }
+
+    @Override
+    public void enterPower(@NotNull Python3Parser.PowerContext ctx) {
+    }
+
+    @Override
+    public void exitPower(@NotNull Python3Parser.PowerContext ctx) {
+    }
+
+    @Override
+    public void enterDotted_as_name(@NotNull Python3Parser.Dotted_as_nameContext ctx) {
+    }
+
+    @Override
+    public void exitDotted_as_name(@NotNull Python3Parser.Dotted_as_nameContext ctx) {
+    }
+
+    @Override
+    public void enterFactor(@NotNull Python3Parser.FactorContext ctx) {
+    }
+
+    @Override
+    public void exitFactor(@NotNull Python3Parser.FactorContext ctx) {
+    }
+
+    @Override
+    public void enterSliceop(@NotNull Python3Parser.SliceopContext ctx) {
+    }
+
+    @Override
+    public void exitSliceop(@NotNull Python3Parser.SliceopContext ctx) {
+    }
+
+    @Override
+    public void enterFuncdef(@NotNull Python3Parser.FuncdefContext ctx) {
+        jplagParser.add(METHOD_BEGIN, ctx.getStart());
+    }
+
+    @Override
+    public void exitFuncdef(@NotNull Python3Parser.FuncdefContext ctx) {
+        jplagParser.add(METHOD_END, ctx.getStart());
+    }
+
+    @Override
+    public void enterSubscriptlist(@NotNull Python3Parser.SubscriptlistContext ctx) {
+    }
+
+    @Override
+    public void exitSubscriptlist(@NotNull Python3Parser.SubscriptlistContext ctx) {
+    }
+
+    @Override
+    public void enterTest_nocond(@NotNull Python3Parser.Test_nocondContext ctx) {
+    }
+
+    @Override
+    public void exitTest_nocond(@NotNull Python3Parser.Test_nocondContext ctx) {
+    }
+
+    @Override
+    public void enterComp_iter(@NotNull Python3Parser.Comp_iterContext ctx) {
+    }
+
+    @Override
+    public void exitComp_iter(@NotNull Python3Parser.Comp_iterContext ctx) {
+    }
+
+    @Override
+    public void enterNonlocal_stmt(@NotNull Python3Parser.Nonlocal_stmtContext ctx) {
+    }
+
+    @Override
+    public void exitNonlocal_stmt(@NotNull Python3Parser.Nonlocal_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterEval_input(@NotNull Python3Parser.Eval_inputContext ctx) {
+    }
+
+    @Override
+    public void exitEval_input(@NotNull Python3Parser.Eval_inputContext ctx) {
+    }
+
+    @Override
+    public void enterVfpdef(@NotNull Python3Parser.VfpdefContext ctx) {
+    }
+
+    @Override
+    public void exitVfpdef(@NotNull Python3Parser.VfpdefContext ctx) {
+    }
+
+    @Override
+    public void enterImport_name(@NotNull Python3Parser.Import_nameContext ctx) {
+    }
+
+    @Override
+    public void exitImport_name(@NotNull Python3Parser.Import_nameContext ctx) {
+    }
+
+    @Override
+    public void enterComp_if(@NotNull Python3Parser.Comp_ifContext ctx) {
+    }
+
+    @Override
+    public void exitComp_if(@NotNull Python3Parser.Comp_ifContext ctx) {
+    }
+
+    @Override
+    public void enterAugassign(@NotNull Python3Parser.AugassignContext ctx) {
+        jplagParser.add(ASSIGN, ctx.getStart());
+    }
+
+    @Override
+    public void exitAugassign(@NotNull Python3Parser.AugassignContext ctx) {
+    }
+
+    @Override
+    public void enterPass_stmt(@NotNull Python3Parser.Pass_stmtContext ctx) {
+    }
+
+    @Override
+    public void exitPass_stmt(@NotNull Python3Parser.Pass_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterExpr_stmt(@NotNull Python3Parser.Expr_stmtContext ctx) {
+    }
+
+    @Override
+    public void exitExpr_stmt(@NotNull Python3Parser.Expr_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterYield_stmt(@NotNull Python3Parser.Yield_stmtContext ctx) {
+        jplagParser.add(YIELD, ctx.getStart());
+    }
+
+    @Override
+    public void exitYield_stmt(@NotNull Python3Parser.Yield_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterSuite(@NotNull Python3Parser.SuiteContext ctx) {
+    }
+
+    @Override
+    public void exitSuite(@NotNull Python3Parser.SuiteContext ctx) {
+    }
+
+    @Override
+    public void enterContinue_stmt(@NotNull Python3Parser.Continue_stmtContext ctx) {
+        jplagParser.add(CONTINUE, ctx.getStart());
+    }
+
+    @Override
+    public void exitContinue_stmt(@NotNull Python3Parser.Continue_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterTestlist_star_expr(@NotNull Python3Parser.Testlist_star_exprContext ctx) {
+    }
+
+    @Override
+    public void exitTestlist_star_expr(@NotNull Python3Parser.Testlist_star_exprContext ctx) {
+    }
+
+    @Override
+    public void enterVarargslist(@NotNull Python3Parser.VarargslistContext ctx) {
+    }
+
+    @Override
+    public void exitVarargslist(@NotNull Python3Parser.VarargslistContext ctx) {
+    }
+
+    @Override
+    public void enterFor_stmt(@NotNull Python3Parser.For_stmtContext ctx) {
+        jplagParser.add(FOR_BEGIN, ctx.getStart());
+    }
+
+    @Override
+    public void exitFor_stmt(@NotNull Python3Parser.For_stmtContext ctx) {
+        jplagParser.add(FOR_END, ctx.getStart());
+    }
+
+    @Override
+    public void enterDel_stmt(@NotNull Python3Parser.Del_stmtContext ctx) {
+        jplagParser.add(DEL, ctx.getStart());
+    }
+
+    @Override
+    public void exitDel_stmt(@NotNull Python3Parser.Del_stmtContext ctx) {
+    }
+
+    @Override
+    public void enterAtom(@NotNull Python3Parser.AtomContext ctx) {
+    }
+
+    @Override
+    public void exitAtom(@NotNull Python3Parser.AtomContext ctx) {
+    }
+
+    @Override
+    public void enterStmt(@NotNull Python3Parser.StmtContext ctx) {
+    }
+
+    @Override
+    public void exitStmt(@NotNull Python3Parser.StmtContext ctx) {
+    }
+
+    @Override
+    public void enterEveryRule(@NotNull ParserRuleContext ctx) {
+    }
+
+    @Override
+    public void exitEveryRule(@NotNull ParserRuleContext ctx) {
+    }
+
+    @Override
+    public void visitTerminal(@NotNull TerminalNode node) {
+        if (node.getText().equals("=")) {
+            jplagParser.add(ASSIGN, node.getSymbol());
+        }else  if (node.getText().equals("finally")) {
+            jplagParser.add(FINALLY, node.getSymbol());
+        }
+    }
+
+    @Override
+    public void visitErrorNode(@NotNull ErrorNode node) {
+    }
+}

jplag.frontend.python-3/src/main/java/jplag/python3/Language.java

@@ -0,0 +1,64 @@
+package jplag.python3;
+
+import java.io.File;
+
+import jplag.ProgramI;
+
+public class Language implements jplag.Language {
+
+    private jplag.python3.Parser parser;
+
+    public Language(ProgramI program) {
+        this.parser = new jplag.python3.Parser();
+        this.parser.setProgram(program);
+    }
+
+    public String[] suffixes() {
+        String[] res = {".py"};
+        return res;
+    }
+
+    public int errorsCount() {
+        return this.parser.errorsCount();
+    }
+
+    public String name() {
+        return "Python3 Parser";
+    }
+
+    public String getShortName() {
+        return "python3";
+    }
+
+    public int min_token_match() {
+        return 9;
+    }
+
+    public jplag.Structure parse(File dir, String[] files) {
+        return this.parser.parse(dir, files);
+    }
+
+    public boolean errors() {
+        return this.parser.getErrors();
+    }
+
+    public boolean supportsColumns() {
+        return false;
+    }
+
+    public boolean isPreformated() {
+        return true;
+    }
+
+    public boolean usesIndex() {
+        return false;
+    }
+
+    public int noOfTokens() {
+        return jplag.python3.Python3Token.numberOfTokens();
+    }
+
+    public String type2string(int type) {
+        return jplag.python3.Python3Token.type2string(type);
+    }
+}

jplag.frontend.python-3/src/main/java/jplag/python3/Parser.java

@@ -0,0 +1,123 @@
+package jplag.python3;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+
+import jplag.Structure;
+import jplag.python3.grammar.Python3Lexer;
+import jplag.python3.grammar.Python3Parser;
+import jplag.python3.grammar.Python3Parser.File_inputContext;
+
+import org.antlr.v4.runtime.ANTLRInputStream;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.tree.ParseTree;
+import org.antlr.v4.runtime.tree.ParseTreeWalker;
+
+public class Parser extends jplag.Parser implements Python3TokenConstants {
+
+    private Structure struct = new Structure();
+    ;
+	private String currentFile;
+
+    public static void main(String args[]) {
+        if (args.length < 1) {
+            System.out.println("Only one ore more files as parameter allowed.");
+            System.exit(-1);
+        }
+        Parser parser = new Parser();
+        parser.setProgram(new jplag.StrippedProgram());
+        jplag.Structure struct = parser.parse(null, args);
+        try {
+            BufferedReader reader = new BufferedReader(new FileReader(new File(args[0])));
+            int lineNr = 1;
+            int token = 0;
+            String line;
+            while ((line = reader.readLine()) != null) {
+                if (token < struct.size()) {
+                    boolean first = true;
+                    while (struct.tokens[token] != null
+                            && struct.tokens[token].getLine() == lineNr) {
+                        if (!first) {
+                            System.out.println();
+                        }
+                        Python3Token tok = (Python3Token) struct.tokens[token];
+                        System.out.print(Python3Token.type2string(tok.type) + " ("
+                                + tok.getLine() + ","
+                                + tok.getColumn() + ","
+                                + tok.getLength() + ")\t");
+                        first = false;
+                        token++;
+                    }
+                    if (first) {
+                        System.out.print("                \t");
+                    }
+                }
+                System.out.println(line);
+                lineNr++;
+            }
+            reader.close();
+        } catch (IOException e) {
+            System.out.println(e);
+        }
+    }
+
+    public jplag.Structure parse(File dir, String files[]) {
+        struct = new Structure();
+        errors = 0;
+        for (int i = 0; i < files.length; i++) {
+            getProgram().print(null, "Parsing file " + files[i] + "\n");
+            if (!parseFile(dir, files[i])) {
+                errors++;
+            }
+            System.gc();//Emeric
+            struct.addToken(new Python3Token(FILE_END, files[i], -1, -1, -1));
+        }
+        this.parseEnd();
+        return struct;
+    }
+
+    public boolean parseFile(File dir, String file) {
+        BufferedInputStream fis;
+
+        ANTLRInputStream input;
+        try {
+            fis = new BufferedInputStream(new FileInputStream(new File(dir, file)));
+            currentFile = file;
+            input = new ANTLRInputStream(fis);
+
+            // create a lexer that feeds off of input CharStream
+            Python3Lexer lexer = new Python3Lexer(input);
+
+            // create a buffer of tokens pulled from the lexer
+            CommonTokenStream tokens = new CommonTokenStream(lexer);
+
+            // create a parser that feeds off the tokens buffer
+            Python3Parser parser = new Python3Parser(tokens);
+            File_inputContext in = parser.file_input();
+
+            ParseTreeWalker ptw = new ParseTreeWalker();
+            for (int i = 0; i < in.getChildCount(); i++) {
+                ParseTree pt = in.getChild(i);
+                ptw.walk(new JplagPython3Listener(this), pt);
+            }
+
+        } catch (IOException e) {
+            getProgram().addError("Parsing Error in '" + file + "':\n" + e.getMessage() + "\n");
+            return false;
+        }
+
+        return true;
+    }
+
+    //	public void addTerminal(int type, int line, int start, String text) {
+    //		struct.addToken(new JavaToken(type, (currentFile == null ? "null" : currentFile), line, start, text.length()));
+    //	}
+    public void add(int type, org.antlr.v4.runtime.Token tok) {
+        struct.addToken(new Python3Token(type, (currentFile == null ? "null" : currentFile), tok.getLine(), tok.getStartIndex(),
+                tok.getText().length()));
+    }
+}

jplag.frontend.python-3/src/main/java/jplag/python3/Python3Token.java

@@ -0,0 +1,111 @@
+package jplag.python3;
+
+public class Python3Token extends jplag.Token implements Python3TokenConstants {
+
+    private int line, column, length;
+
+    public Python3Token(int type, String file, int line, int column, int length) {
+        super(type, file, line, column, length);
+    }
+
+    public int getLine() {
+        return line;
+    }
+
+    public int getColumn() {
+        return column;
+    }
+
+    public int getLength() {
+        return length;
+    }
+
+    public void setLine(int line) {
+        this.line = line;
+    }
+
+    public void setColumn(int column) {
+        this.column = column;
+    }
+
+    public void setLength(int length) {
+        this.length = length;
+    }
+
+    public static String type2string(int type) {
+        switch (type) {
+            case Python3TokenConstants.FILE_END:
+                return "********";
+            case Python3TokenConstants.SEPARATOR_TOKEN:
+                return "METHOD_SEPARATOR";
+
+            case IMPORT:
+                return "IMPORT  ";
+            case CLASS_BEGIN:
+                return "CLASS{  ";
+            case CLASS_END:
+                return "}CLASS  ";
+            case METHOD_BEGIN:
+                return "METHOD{ ";
+            case METHOD_END:
+                return "}METHOD ";
+            case ASSIGN:
+                return "ASSIGN  ";
+            case WHILE_BEGIN:
+                return "WHILE{  ";
+            case WHILE_END:
+                return "}WHILE  ";
+            case FOR_BEGIN:
+                return "FOR{    ";
+            case FOR_END:
+                return "}FOR    ";
+            case TRY_BEGIN:
+                return "TRY{    ";
+            case EXCEPT_BEGIN:
+                return "CATCH{  ";
+            case EXCEPT_END:
+                return "}CATCH  ";
+            case FINALLY:
+                return "FINALLY ";
+            case IF_BEGIN:
+                return "IF{     ";
+            case IF_END:
+                return "}IF     ";
+            case APPLY:
+                return "APPLY   ";
+            case BREAK:
+                return "BREAK   ";
+            case CONTINUE:
+                return "CONTINUE";
+            case RETURN:
+                return "RETURN  ";
+            case RAISE:
+                return "RAISE   ";
+            case DEC_BEGIN:
+                return "DECOR{  ";
+            case DEC_END:
+                return "}DECOR  ";
+            case LAMBDA:
+                return "LAMBDA  ";
+            case ARRAY:
+                return "ARRAY   ";
+            case ASSERT:
+                return "ASSERT  ";
+            case YIELD:
+                return "YIELD   ";
+            case DEL:
+                return "DEL     ";
+            case WITH_BEGIN:
+                return "WITH{   ";
+            case WITH_END:
+                return "}WITH   ";
+            default:
+                System.err.println("*UNKNOWN: " + type);
+                return "*UNKNOWN" + type;
+        }
+    }
+
+    public static int numberOfTokens() {
+        return NUM_DIFF_TOKENS;
+    }
+}

jplag.frontend.python-3/src/main/java/jplag/python3/Python3TokenConstants.java

@@ -0,0 +1,40 @@
+package jplag.python3;
+
+public interface Python3TokenConstants extends jplag.TokenConstants {
+
+    final static int FILE_END = 0;
+    final static int SEPARATOR_TOKEN = 1;
+
+    final static int IMPORT = 2;
+    final static int CLASS_BEGIN = 3;
+    final static int CLASS_END = 4;
+    final static int METHOD_BEGIN = 5;
+    final static int METHOD_END = 6;
+    final static int ASSIGN = 7;
+    final static int WHILE_BEGIN = 8;
+    final static int WHILE_END = 9;
+    final static int FOR_BEGIN = 10;
+    final static int FOR_END = 11;
+    final static int TRY_BEGIN = 12;
+    final static int EXCEPT_BEGIN = 13;
+    final static int EXCEPT_END = 14;
+    final static int FINALLY = 15;
+    final static int IF_BEGIN = 16;
+    final static int IF_END = 18;
+    final static int APPLY = 19;
+    final static int BREAK = 20;
+    final static int CONTINUE = 21;
+    final static int RETURN = 22;
+    final static int RAISE = 23;
+    final static int DEC_BEGIN = 24;
+    final static int DEC_END = 25;
+    final static int LAMBDA = 26;
+    final static int ARRAY = 27;
+    final static int ASSERT = 28;
+    final static int YIELD = 29;
+    final static int DEL = 30;
+    final static int WITH_BEGIN = 31;
+    final static int WITH_END = 32;
+
+    final static int NUM_DIFF_TOKENS = 33;
+}

jplag.parent/pom.xml

@@ -214,6 +214,11 @@
 				<artifactId>java-1.7</artifactId>
 				<version>0.0.1-SNAPSHOT</version>
 			</dependency>
+			<dependency>
+				<groupId>edu.kit.ipd.jplag</groupId>
+				<artifactId>python-3</artifactId>
+				<version>0.0.1-SNAPSHOT</version>
+			</dependency>
 			<dependency>
 				<groupId>edu.kit.ipd.jplag</groupId>
 				<artifactId>csharp-1.2</artifactId>

jplag/pom.xml

@@ -206,6 +206,10 @@
 			<groupId>edu.kit.ipd.jplag</groupId>
 			<artifactId>java-1.7</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>edu.kit.ipd.jplag</groupId>
+			<artifactId>python-3</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>edu.kit.ipd.jplag</groupId>
 			<artifactId>csharp-1.2</artifactId>

jplag/src/main/java/jplag/options/Options.java

@@ -32,6 +32,7 @@ public abstract class Options {
 			"java15dm", "jplag.java15.LanguageWithDelimitedMethods",
 			"java12", "jplag.java.Language",
 			"java11", "jplag.javax.Language",
+			"python3","jplag.python3.Language",
 			"c/c++", "jplag.cpp.Language",
 			"c#-1.2", "jplag.csharp.Language",
 			"char", "jplag.chars.Language",

pom.xml

@@ -17,6 +17,7 @@
 		<module>jplag.frontend.java-1.2</module>
 		<module>jplag.frontend.java-1.5</module>
 		<module>jplag.frontend.java-1.7</module>
+		<module>jplag.frontend.python-3</module>
 		<module>jplag.frontend.scheme</module>
 		<module>jplag.frontend.text</module>
 		<module>jplag.utils</module>