EndToEndSuiteTest.java 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258
  1. package de.jplag.endtoend;
  2. import static org.junit.jupiter.api.Assertions.assertNotNull;
  3. import static org.junit.jupiter.api.Assertions.assertTrue;
  4. import java.io.IOException;
  5. import java.nio.file.Path;
  6. import java.util.ArrayList;
  7. import java.util.Arrays;
  8. import java.util.Collection;
  9. import java.util.HashMap;
  10. import java.util.List;
  11. import java.util.Map;
  12. import java.util.Map.Entry;
  13. import java.util.Optional;
  14. import java.util.StringJoiner;
  15. import org.junit.jupiter.api.AfterAll;
  16. import org.junit.jupiter.api.DynamicTest;
  17. import org.junit.jupiter.api.TestFactory;
  18. import de.jplag.JPlag;
  19. import de.jplag.JPlagComparison;
  20. import de.jplag.JPlagResult;
  21. import de.jplag.LanguageLoader;
  22. import de.jplag.endtoend.helper.FileHelper;
  23. import de.jplag.endtoend.helper.JsonHelper;
  24. import de.jplag.endtoend.helper.TestSuiteHelper;
  25. import de.jplag.endtoend.model.*;
  26. import de.jplag.exceptions.ExitException;
  27. import de.jplag.options.JPlagOptions;
  28. /**
  29. * Main test class for end-to-end testing in all language. The test cases aim to detect changes in the detection of
  30. * plagiarism in the Java language and to be able to roughly categorize them. The plagiarism is compared with the
  31. * original class. The results are compared with the results from previous tests and changes are detected.
  32. */
  33. public class EndToEndSuiteTest {
  34. // Language -> directory names and Paths
  35. private Map<String, Map<String, Path>> LanguageToTestCaseMapper;
  36. private List<Options> options;
  37. private static Map<String, List<ResultDescription>> temporaryResultList;
  38. private static List<String> validationErrors;
  39. private static String languageIdentifier;
  40. public EndToEndSuiteTest() throws IOException {
  41. // Loading the test resources
  42. LanguageToTestCaseMapper = TestSuiteHelper.getAllLanguageResources();
  43. // creating the temporary lists for the test run
  44. validationErrors = new ArrayList<>();
  45. temporaryResultList = new HashMap<>();
  46. // creating options object for the testSuite
  47. setRunOptions();
  48. }
  49. /**
  50. * creating the required options object for the endToEnd tests
  51. */
  52. private void setRunOptions() {
  53. options = new ArrayList<>();
  54. options.add(new Options(9));
  55. options.add(new Options(3));
  56. }
  57. /**
  58. * Creates the result json files based on the current test results after the test run.
  59. * @throws IOException is thrown for all problems that may occur while parsing the json file. This includes both reading
  60. */
  61. @AfterAll
  62. public static void tearDown() throws IOException {
  63. for (var resultDescriptionItem : temporaryResultList.entrySet()) {
  64. JsonHelper.writeJsonModelsToJsonFile(resultDescriptionItem.getValue(), resultDescriptionItem.getKey(), languageIdentifier);
  65. }
  66. }
  67. /**
  68. * Creates the test cases over all language options for which data is available and the current test options.
  69. * @return dynamic test cases across all test data and languages
  70. * @throws IOException is thrown for all problems that may occur while parsing the json file. This includes both reading
  71. */
  72. @TestFactory
  73. Collection<DynamicTest> dynamicOverAllTest() throws IOException {
  74. for (Entry<String, Map<String, Path>> languageMap : LanguageToTestCaseMapper.entrySet()) {
  75. String currentLanguageIdentifier = languageMap.getKey();
  76. for (Entry<String, Path> languagePaths : languageMap.getValue().entrySet()) {
  77. String[] fileNames = FileHelper.loadAllTestFileNames(languagePaths.getValue());
  78. var testCases = TestSuiteHelper.getTestCases(fileNames, languagePaths.getValue());
  79. var testCollection = new ArrayList<DynamicTest>();
  80. String directoryName = languagePaths.getValue().getFileName().toString();
  81. List<ResultDescription> tempResult = JsonHelper.getJsonModelListFromPath(directoryName, currentLanguageIdentifier);
  82. languageIdentifier = currentLanguageIdentifier;
  83. for (Options option : options) {
  84. for (var testCase : testCases) {
  85. Optional<ResultDescription> currentResultDescription = tempResult.stream().filter(x -> x.options().equals(option))
  86. .findFirst();
  87. testCollection.add(DynamicTest.dynamicTest(getTestCaseDisplayName(option, languageIdentifier, testCase), () -> {
  88. runTests(directoryName, option, currentLanguageIdentifier, testCase, currentResultDescription);
  89. }));
  90. }
  91. }
  92. return testCollection;
  93. }
  94. }
  95. return null;
  96. }
  97. /**
  98. * Superordinate test function to be able to continue to check all data to be tested in case of failed tests
  99. * @param directoryName name of the current tested directory
  100. * @param option for the current test run
  101. * @param currentLanguageIdentifier current JPlag language option
  102. * @param testFiles files to be tested
  103. * @param currentResultDescription results stored for the test data
  104. * @throws IOException Signals that an I/O exception of some sort has occurred. Thisclass is the general class of
  105. * exceptions produced by failed orinterrupted I/O operations
  106. * @throws ExitException Exceptions for problems during the execution of JPlag that lead to an preemptive exit.
  107. */
  108. private void runTests(String directoryName, Options option, String currentLanguageIdentifier, String[] testFiles,
  109. Optional<ResultDescription> currentResultDescription) throws IOException, ExitException {
  110. try {
  111. ResultDescription currentResult = currentResultDescription.orElse(null);
  112. runJPlagTestSuite(directoryName, option, currentLanguageIdentifier, testFiles, currentResult);
  113. } finally {
  114. validationErrors.clear();
  115. TestSuiteHelper.clear();
  116. }
  117. }
  118. /**
  119. * EndToEnd test for the passed objects
  120. * @param directoryName name of the current tested directory
  121. * @param options for the current test run
  122. * @param languageIdentifier current JPlag language option
  123. * @param testFiles files to be tested
  124. * @param currentResultDescription results stored for the test data
  125. * @throws IOException Signals that an I/O exception of some sort has occurred. Thisclass is the general class of
  126. * exceptions produced by failed orinterrupted I/O operations
  127. * @throws ExitException Exceptions for problems during the execution of JPlag that lead to an preemptive exit.
  128. */
  129. private void runJPlagTestSuite(String directoryName, Options options, String languageIdentifier, String[] testFiles,
  130. ResultDescription currentResultDescription) throws IOException, ExitException {
  131. String[] submissionPath = FileHelper.createNewTestCaseDirectory(testFiles);
  132. var language = LanguageLoader.getLanguage(languageIdentifier).orElseThrow();
  133. JPlagOptions jplagOptions = new JPlagOptions(language, Arrays.asList(submissionPath), new ArrayList<>())
  134. .withMinimumTokenMatch(options.minimumTokenMatch());
  135. JPlagResult jplagResult = new JPlag(jplagOptions).run();
  136. List<JPlagComparison> currentJPlagComparison = jplagResult.getAllComparisons();
  137. for (JPlagComparison jPlagComparison : currentJPlagComparison) {
  138. String identifier = TestSuiteHelper.getTestIdentifier(jPlagComparison);
  139. addToTemporaryResultMap(directoryName, options, jPlagComparison, languageIdentifier);
  140. assertNotNull(currentResultDescription, "No stored result could be found for the current LanguageOption! " + options.toString());
  141. ExpectedResult result = currentResultDescription.getExpectedResultByIdentifier(TestSuiteHelper.getTestIdentifier(jPlagComparison));
  142. assertNotNull(result, "No stored result could be found for the identifier! " + identifier);
  143. if (Float.compare(result.resultSimilarityMinimum(), jPlagComparison.minimalSimilarity()) != 0) {
  144. addToValidationErrors("minimalSimilarity", String.valueOf(result.resultSimilarityMinimum()),
  145. String.valueOf(jPlagComparison.minimalSimilarity()));
  146. }
  147. if (Float.compare(result.resultSimilarityMaximum(), jPlagComparison.maximalSimilarity()) != 0) {
  148. addToValidationErrors("maximalSimilarity", String.valueOf(result.resultSimilarityMaximum()),
  149. String.valueOf(jPlagComparison.maximalSimilarity()));
  150. }
  151. if (Integer.compare(result.resultMatchedTokenNumber(), jPlagComparison.getNumberOfMatchedTokens()) != 0) {
  152. addToValidationErrors("numberOfMatchedTokens", String.valueOf(result.resultMatchedTokenNumber()),
  153. String.valueOf(jPlagComparison.getNumberOfMatchedTokens()));
  154. }
  155. assertTrue(validationErrors.isEmpty(), createValidationErrorOutout());
  156. }
  157. }
  158. /**
  159. * Creates the display message for failed tests
  160. * @param valueName Name of the failed test object
  161. * @param currentValue current test values
  162. * @param expectedValue expected test values
  163. */
  164. private void addToValidationErrors(String valueName, String currentValue, String expectedValue) {
  165. validationErrors.add(valueName + " was " + currentValue + " but expected " + expectedValue);
  166. }
  167. /**
  168. * Creates the display info from the current failed test results
  169. * @return formatted text for the failed comparative values of the current test
  170. */
  171. private String createValidationErrorOutout() {
  172. StringJoiner joiner = new StringJoiner(System.lineSeparator());
  173. joiner.add(""); // empty line at start
  174. joiner.add("There were <" + validationErrors.size() + "> validation error(s):");
  175. validationErrors.stream().forEach(errorLine -> joiner.add(errorLine));
  176. joiner.add(""); // line break at the end
  177. return joiner.toString();
  178. }
  179. /**
  180. * Add or create the current test results in a temporary list to be able to save them later.
  181. * @param directoryName name of the current tested directory
  182. * @param options for the current test run
  183. * @param jPlagComparison current test results
  184. * @param languageIdentifier current JPlag language option
  185. */
  186. private void addToTemporaryResultMap(String directoryName, Options options, JPlagComparison jPlagComparison, String languageIdentifier) {
  187. var element = temporaryResultList.get(directoryName);
  188. if (element != null) {
  189. for (var item : element) {
  190. if (item.options().equals(options)) {
  191. item.putIdentifierToResultMap(TestSuiteHelper.getTestIdentifier(jPlagComparison), new ExpectedResult(
  192. jPlagComparison.minimalSimilarity(), jPlagComparison.maximalSimilarity(), jPlagComparison.getNumberOfMatchedTokens()));
  193. return;
  194. }
  195. }
  196. Map<String, ExpectedResult> temporaryHashMap = new HashMap<>();
  197. temporaryHashMap.put(TestSuiteHelper.getTestIdentifier(jPlagComparison), new ExpectedResult(jPlagComparison.minimalSimilarity(),
  198. jPlagComparison.maximalSimilarity(), jPlagComparison.getNumberOfMatchedTokens()));
  199. element.add(new ResultDescription(languageIdentifier, options, temporaryHashMap));
  200. } else {
  201. var temporaryNewResultList = new ArrayList<ResultDescription>();
  202. Map<String, ExpectedResult> temporaryHashMap = new HashMap<>();
  203. temporaryHashMap.put(TestSuiteHelper.getTestIdentifier(jPlagComparison), new ExpectedResult(jPlagComparison.minimalSimilarity(),
  204. jPlagComparison.maximalSimilarity(), jPlagComparison.getNumberOfMatchedTokens()));
  205. temporaryNewResultList.add(new ResultDescription(languageIdentifier, options, temporaryHashMap));
  206. temporaryResultList.put(directoryName, temporaryNewResultList);
  207. }
  208. }
  209. /**
  210. * Creates the name of the test for better assignment and readability Pattern: Language: (option values)
  211. * filename1-filename2
  212. * @param option under which the current test run
  213. * @param languageIdentifier current language used in the test
  214. * @param testFiles test data for assigning by filename
  215. * @return display name for the individual tests
  216. */
  217. private String getTestCaseDisplayName(Options option, String languageIdentifier, String[] testFiles) {
  218. StringBuilder stringBuilder = new StringBuilder();
  219. stringBuilder.append("(" + String.valueOf(option.minimumTokenMatch()) + ")");
  220. for (int counter = 0; counter < testFiles.length; counter++) {
  221. String fileName = Path.of(testFiles[counter]).toFile().getName();
  222. stringBuilder.append(fileName.substring(0, fileName.lastIndexOf('.')));
  223. if (counter + 1 < testFiles.length) {
  224. stringBuilder.append("-");
  225. }
  226. }
  227. return languageIdentifier + ": " + stringBuilder.toString();
  228. }
  229. }