Parser.java 2.9 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788
  1. package de.jplag.python3;
  2. import java.io.BufferedInputStream;
  3. import java.io.File;
  4. import java.io.FileInputStream;
  5. import java.io.IOException;
  6. import org.antlr.v4.runtime.CharStream;
  7. import org.antlr.v4.runtime.CharStreams;
  8. import org.antlr.v4.runtime.CommonTokenStream;
  9. import org.antlr.v4.runtime.Token;
  10. import org.antlr.v4.runtime.tree.ParseTree;
  11. import org.antlr.v4.runtime.tree.ParseTreeWalker;
  12. import de.jplag.AbstractParser;
  13. import de.jplag.TokenList;
  14. import de.jplag.python3.grammar.Python3Lexer;
  15. import de.jplag.python3.grammar.Python3Parser;
  16. import de.jplag.python3.grammar.Python3Parser.File_inputContext;
  17. public class Parser extends AbstractParser {
  18. private TokenList tokens = new TokenList();
  19. private String currentFile;
  20. /**
  21. * Creates the parser.
  22. */
  23. public Parser() {
  24. super();
  25. }
  26. public TokenList parse(File directory, String[] files) {
  27. tokens = new TokenList();
  28. errors = 0;
  29. for (String file : files) {
  30. logger.trace("Parsing file {}", file);
  31. if (!parseFile(directory, file)) {
  32. errors++;
  33. }
  34. tokens.addToken(new Python3Token(Python3TokenConstants.FILE_END, file, -1, -1, -1));
  35. }
  36. return tokens;
  37. }
  38. private boolean parseFile(File directory, String file) {
  39. BufferedInputStream inputStream;
  40. CharStream input;
  41. try {
  42. inputStream = new BufferedInputStream(new FileInputStream(new File(directory, file)));
  43. currentFile = file;
  44. input = CharStreams.fromStream(inputStream);
  45. // create a lexer that feeds off of input CharStream
  46. Python3Lexer lexer = new Python3Lexer(input);
  47. // create a buffer of tokens pulled from the lexer
  48. CommonTokenStream tokens = new CommonTokenStream(lexer);
  49. // create a parser that feeds off the tokens buffer
  50. Python3Parser parser = new Python3Parser(tokens);
  51. File_inputContext in = parser.file_input();
  52. ParseTreeWalker ptw = new ParseTreeWalker();
  53. for (int i = 0; i < in.getChildCount(); i++) {
  54. ParseTree pt = in.getChild(i);
  55. ptw.walk(new JplagPython3Listener(this), pt);
  56. }
  57. } catch (IOException e) {
  58. logger.error("Parsing Error in '" + file + "': " + e.getMessage(), e);
  59. return false;
  60. }
  61. return true;
  62. }
  63. public void add(int type, Token token) {
  64. tokens.addToken(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(), token.getCharPositionInLine() + 1,
  65. token.getText().length()));
  66. }
  67. public void addEnd(int type, Token token) {
  68. tokens.addToken(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(),
  69. tokens.getToken(tokens.size() - 1).getColumn() + 1, 0));
  70. }
  71. }