Parser.java 2.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990
  1. package de.jplag.python3;
  2. import java.io.BufferedInputStream;
  3. import java.io.File;
  4. import java.io.FileInputStream;
  5. import java.io.IOException;
  6. import java.util.ArrayList;
  7. import java.util.List;
  8. import org.antlr.v4.runtime.CharStream;
  9. import org.antlr.v4.runtime.CharStreams;
  10. import org.antlr.v4.runtime.CommonTokenStream;
  11. import org.antlr.v4.runtime.Token;
  12. import org.antlr.v4.runtime.tree.ParseTree;
  13. import org.antlr.v4.runtime.tree.ParseTreeWalker;
  14. import de.jplag.AbstractParser;
  15. import de.jplag.TokenConstants;
  16. import de.jplag.python3.grammar.Python3Lexer;
  17. import de.jplag.python3.grammar.Python3Parser;
  18. import de.jplag.python3.grammar.Python3Parser.File_inputContext;
  19. public class Parser extends AbstractParser {
  20. private List<de.jplag.Token> tokens;
  21. private String currentFile;
  22. /**
  23. * Creates the parser.
  24. */
  25. public Parser() {
  26. super();
  27. }
  28. public List<de.jplag.Token> parse(File directory, String[] files) {
  29. tokens = new ArrayList<>();
  30. errors = 0;
  31. for (String file : files) {
  32. logger.trace("Parsing file {}", file);
  33. if (!parseFile(directory, file)) {
  34. errors++;
  35. }
  36. tokens.add(new Python3Token(TokenConstants.FILE_END, file, -1, -1, -1));
  37. }
  38. return tokens;
  39. }
  40. private boolean parseFile(File directory, String file) {
  41. BufferedInputStream inputStream;
  42. CharStream input;
  43. try {
  44. inputStream = new BufferedInputStream(new FileInputStream(new File(directory, file)));
  45. currentFile = file;
  46. input = CharStreams.fromStream(inputStream);
  47. // create a lexer that feeds off of input CharStream
  48. Python3Lexer lexer = new Python3Lexer(input);
  49. // create a buffer of tokens pulled from the lexer
  50. CommonTokenStream tokens = new CommonTokenStream(lexer);
  51. // create a parser that feeds off the tokens buffer
  52. Python3Parser parser = new Python3Parser(tokens);
  53. File_inputContext in = parser.file_input();
  54. ParseTreeWalker ptw = new ParseTreeWalker();
  55. for (int i = 0; i < in.getChildCount(); i++) {
  56. ParseTree pt = in.getChild(i);
  57. ptw.walk(new JplagPython3Listener(this), pt);
  58. }
  59. } catch (IOException e) {
  60. logger.error("Parsing Error in '" + file + "': " + e.getMessage(), e);
  61. return false;
  62. }
  63. return true;
  64. }
  65. public void add(int type, Token token) {
  66. tokens.add(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(), token.getCharPositionInLine() + 1,
  67. token.getText().length()));
  68. }
  69. public void addEnd(int type, Token token) {
  70. tokens.add(new Python3Token(type, (currentFile == null ? "null" : currentFile), token.getLine(),
  71. tokens.get(tokens.size() - 1).getColumn() + 1, 0));
  72. }
  73. }