Merge branch 'master' into parallel-compare

README.md

@@ -58,9 +58,9 @@ named arguments:
   -x X                   All files named in this file will be ignored in the comparison (line-separated list)
   -t T                   Tune the sensitivity of the comparison. A smaller <n> increases the sensitivity
   -m M                   Match similarity Threshold [0-100]: All matches above this threshold will be saved (Standard: 0.0)
+  -n N                   Maximum number of matches that will be saved. If set to -1 all matches will be saved (Standard: 30)
   -r R                   Name of directory in which the comparison results will be stored (Standard: result)
   -c {normal,parallel}   Comparison mode used to compare the programs (Standard: normal)
-
 ```
 
 ### Java API

jplag/src/main/java/jplag/CLI.java

@@ -4,6 +4,7 @@ import static jplag.strategy.ComparisonMode.NORMAL;
 import static net.sourceforge.argparse4j.impl.Arguments.storeTrue;
 
 import java.io.File;
+import java.util.Random;
 
 import jplag.options.JPlagOptions;
 import jplag.options.LanguageOption;
@@ -21,7 +22,21 @@ import net.sourceforge.argparse4j.inf.Namespace;
  */
 public class CLI {
 
-    private static final String DESCRIPTION = "JPlag - Detecting Software Plagiarism";
+    private static final String[] DESCRIPTIONS = {
+            "Detecting Software Plagiarism",
+            "Software-Archaeological Playground",
+            "Since 1994",
+            "Scientifically Published",
+            "Maintained by SDQ",
+            "RIP Structure and Table",
+            "What else?",
+            "You have been warned",
+            "Since Java 1.0",
+            "More Abstract than Tree",
+            "Students Nightmare",
+            "No, changing variable names does not work"
+    };
+
     private static final String PROGRAM_NAME = "jplag";
 
     // TODO SH: Replace verbosity when integrating a real logging library
@@ -54,7 +69,7 @@ public class CLI {
      * Creates the command line interface and initializes the argument parser.
      */
     public CLI() {
-        parser = ArgumentParsers.newFor(PROGRAM_NAME).build().defaultHelp(true).description(DESCRIPTION);
+        parser = ArgumentParsers.newFor(PROGRAM_NAME).build().defaultHelp(true).description(generateDescription());
 
         // TODO SH: Fix code duplication of CLI arguments
         parser.addArgument("rootDir").help("The root-directory that contains all submissions");
@@ -67,7 +82,8 @@ public class CLI {
         parser.addArgument("-p").help("comma-separated list of all filename suffixes that are included");
         parser.addArgument("-x").help("All files named in this file will be ignored in the comparison (line-separated list)");
         parser.addArgument("-t").help("Tune the sensitivity of the comparison. A smaller <n> increases the sensitivity");
-        parser.addArgument("-m").setDefault(0f).help("Match similarity Threshold [0-100]: All matches above this threshold will be saved");
+        parser.addArgument("-m").setDefault(0f).help("Match similarity threshold [0-100]: All matches above this threshold will be saved");
+        parser.addArgument("-n").setDefault(30).help("Maximum number of matches that will be saved. If set to -1 all matches will be saved");
         parser.addArgument("-r").setDefault("result").help("Name of directory in which the comparison results will be stored");
         parser.addArgument("-c").choices(ComparisonMode.allNames()).setDefault(NORMAL.getName()).help("Comparison mode used to compare the programs");
     }
@@ -126,10 +142,25 @@ public class CLI {
                 options.setSimilarityThreshold(Float.parseFloat(similarityThreshold));
             } catch (NumberFormatException e) {
                 System.out.println("Illegal similarity threshold. Taking 0 as default value.");
-                options.setSimilarityThreshold(0);
+                options.setSimilarityThreshold(0); // TODO SH: Remove code duplication
+            }
+        }
+
+        String maxNumberOfMatches = namespace.getString("n");
+        if(maxNumberOfMatches != null) {
+            try {
+                options.setMaxNumberOfMatches(Integer.parseInt(maxNumberOfMatches));
+            } catch (NumberFormatException e) {
+                System.out.println("Illegal maximum number of matches. Taking 30 as default value.");
+                options.setMaxNumberOfMatches(30); // TODO SH: Remove code duplication
             }
         }
 
         return options;
     }
+
+    private String generateDescription() {
+        var randomDescription = DESCRIPTIONS[new Random().nextInt(DESCRIPTIONS.length)];
+        return String.format("JPlag - %s", randomDescription);
+    }
 }

jplag/src/main/java/jplag/JPlagResult.java

@@ -68,11 +68,42 @@ public class JPlagResult {
         return similarityDistribution;
     }
 
+    /**
+     * @return a list of all comparisons sorted by percentage (descending).
+     */
+    public List<JPlagComparison> getAllComparisons() {
+        return comparisons;
+    }
+
     /**
      * @return a list of comparisons sorted by percentage (descending).
+     * the maximum number of comparisons is limited by the settings set in JPlagOptions
      */
     public List<JPlagComparison> getComparisons() {
-        return comparisons;
+        if(this.options.getMaxNumberOfMatches() == -1) {
+            return this.getAllComparisons();
+        } else {
+            return this.getComparisons(this.options.getMaxNumberOfMatches());
+        }
+    }
+
+    /**
+     * Returns the first n comparisons (sorted by percentage, descending), limited by the specified parameter.
+     * @param maxCount the maximum number of yield comparisons
+     * @return a list of comparisons with a size of maxCount or less
+     */
+    public List<JPlagComparison> getComparisons(int maxCount) {
+        return comparisons.subList(0, Math.min(maxCount, comparisons.size()));
+    }
+
+    /**
+     * Drops elements from the comparison list to free memory. Note, that this affects the similarity distribution
+     * and is only meant to be used if you don't need the information about comparisons with lower match percentage
+     * anymore.
+     * @param limit the number of comparisons to keep in the list
+     */
+    public void dropComparisons(int limit) {
+        this.comparisons = this.getComparisons(limit);
     }
 
     public long getDuration() {

jplag/src/main/java/jplag/options/JPlagOptions.java

@@ -12,21 +12,6 @@ public class JPlagOptions {
      */
     private Language language;
 
-    /**
-     * Deprecated - use similarityThreshold instead! Maximum number of comparisons to store per run.
-     */
-    @Deprecated
-    private int storeMatches = 30;
-
-    /**
-     * Deprecated - use similarityThreshold instead!
-     * <p>
-     * True, if `storeMatches` should be interpreted as a percentage threshold for the similarity of a comparison; false
-     * otherwise;
-     */
-    @Deprecated
-    private boolean storePercent = false;
-
     /**
      * TODO PB: Decide what to do with this.
      * <p>
@@ -59,6 +44,12 @@ public class JPlagOptions {
      */
     private float similarityThreshold = 0;
 
+    /**
+     * The maximum number of matches that will be saved. This does affect the generated report as well as the internally
+     * saved comparisons. If set to -1 all matches will be saved.
+     */
+    private int maxNumberOfMatches = 30;
+
     /**
      * TODO PB: Not happy with the name yet.
      */
@@ -151,16 +142,6 @@ public class JPlagOptions {
         return fileSuffixes;
     }
 
-    @Deprecated
-    public int getStoreMatches() {
-        return storeMatches;
-    }
-
-    @Deprecated
-    public boolean isStorePercent() {
-        return storePercent;
-    }
-
     public Language getLanguage() {
         return language;
     }
@@ -197,6 +178,10 @@ public class JPlagOptions {
         return similarityThreshold;
     }
 
+    public int getMaxNumberOfMatches() {
+        return this.maxNumberOfMatches;
+    }
+
     public SimilarityMetric getSimilarityMetric() {
         return similarityMetric;
     }
@@ -221,16 +206,6 @@ public class JPlagOptions {
         this.fileSuffixes = fileSuffixes;
     }
 
-    @Deprecated
-    public void setStoreMatches(int storeMatches) {
-        this.storeMatches = storeMatches;
-    }
-
-    @Deprecated
-    public void setStorePercent(boolean storePercent) {
-        this.storePercent = storePercent;
-    }
-
     public void setMinTokenMatch(Integer minTokenMatch) {
         this.minTokenMatch = minTokenMatch;
     }
@@ -269,6 +244,14 @@ public class JPlagOptions {
         }
     }
 
+    public void setMaxNumberOfMatches(int maxNumberOfMatches) {
+        if(maxNumberOfMatches < -1) {
+            this.maxNumberOfMatches = -1;
+        } else {
+            this.maxNumberOfMatches = maxNumberOfMatches;
+        }
+    }
+
     public void setSimilarityMetric(SimilarityMetric similarityMetric) {
         this.similarityMetric = similarityMetric;
     }

jplag/src/main/java/jplag/reporting/Report.java

@@ -445,7 +445,8 @@ public class Report {
         if (result.getComparisons().size() > 0) {
             htmlFile.println("<TR BGCOLOR=#aaaaff VALIGN=top><TD>" + msg.getString("Report.Matches_displayed") + ":</TD>" + "<TD>");
 
-            htmlFile.println(result.getComparisons().size() + " (" + msg.getString("Report.Treshold") + ": "
+            htmlFile.println(result.getComparisons().size() + " of " + result.getAllComparisons().size()
+                    + " (" + msg.getString("Report.Treshold") + ": "
                     + result.getOptions().getSimilarityThreshold() + "%)<br>");
 
             htmlFile.println("</TD></TR>");

jplag/src/test/java/jplag/NormalComparisonTest.java

@@ -2,10 +2,15 @@ package jplag;
 
 import org.junit.Test;
 
+import java.util.Optional;
+
 import static org.junit.Assert.assertEquals;
 
 public class NormalComparisonTest extends TestBase {
 
+    /**
+     * The simple duplicate contains obvious plagiarism.
+     */
     @Test
     public void testSimpleDuplicate() throws ExitException {
         JPlagResult result = runJPlagWithDefaultOptions("SimpleDuplicate");
@@ -14,6 +19,79 @@ public class NormalComparisonTest extends TestBase {
         assertEquals(1, result.getComparisons().size());
         assertEquals(1, result.getComparisons().get(0).matches.size());
         assertEquals(1, result.getSimilarityDistribution()[6]);
+        assertEquals(62.07f, result.getComparisons().get(0).percent(), 0.1f);
     }
 
+    /**
+     * The classes in no duplicate have nearly nothing in common.
+     */
+    @Test
+    public void testNoDuplicate() throws ExitException {
+        JPlagResult result = runJPlagWithDefaultOptions("NoDuplicate");
+
+        assertEquals(3, result.getNumberOfSubmissions());
+        assertEquals(3, result.getComparisons().size());
+
+        result.getAllComparisons().forEach(comparison -> {
+            assertEquals(0f, comparison.percent(), 0.1f);
+        });
+    }
+
+    /**
+     * This case is more complex and consists out of 5 submissions with different plagiarism.
+     * A is the original code (coming from an older JPlag version)
+     * B is a partial copy of that code
+     * C is a full copy of that code
+     * D is dumb plagiarism, e.g., changed variable names, additional unneeded code, ...
+     * E is just a Hello World Java program
+     */
+    @Test
+    public void partialPlagiarism() throws ExitException {
+        JPlagResult result = runJPlagWithDefaultOptions("PartialPlagiarism");
+
+        assertEquals(5, result.getNumberOfSubmissions());
+        assertEquals(10, result.getAllComparisons().size());
+
+        // All comparisons with E shall have no matches
+        result.getAllComparisons()
+                .stream()
+                .filter(comparison ->
+                        comparison.secondSubmission.name.equals("E") ||
+                                comparison.firstSubmission.name.equals("E"))
+                .forEach(comparison ->
+                        assertEquals(0f, comparison.percent(), 0.1f)
+                );
+
+        // Hard coded assertions on selected comparisons
+        assertEquals(24.6f, getSelectedPercent(result, "A", "B"), 0.1f);
+        assertEquals(99.7f, getSelectedPercent(result, "A", "C"), 0.1f);
+        assertEquals(77.9f, getSelectedPercent(result, "A", "D"), 0.1f);
+        assertEquals(24.6f, getSelectedPercent(result, "B", "C"), 0.1f);
+        assertEquals(28.3f, getSelectedPercent(result, "B", "D"), 0.1f);
+        assertEquals(77.9f, getSelectedPercent(result, "C", "D"), 0.1f);
+
+        // More detailed assertions for the plagiarism in A-D
+        var biggestMatch = getSelectedComparison(result, "A", "D");
+        assertEquals(96.4f, biggestMatch.get().percentMaxAB(), 0.1f);
+        assertEquals(65.3f, biggestMatch.get().percentMinAB(), 0.1f);
+        assertEquals(12, biggestMatch.get().matches.size());
+
+    }
+
+    // TODO SH: Methods like this should be moved to the API and also should accept wildcards
+    private float getSelectedPercent(JPlagResult result, String nameA, String nameB) {
+        return getSelectedComparison(result, nameA, nameB)
+                .map(JPlagComparison::percent)
+                .orElse(-1f);
+    }
+
+    private Optional<JPlagComparison> getSelectedComparison(JPlagResult result, String nameA, String nameB) {
+        return result.getAllComparisons().stream()
+                .filter(comparison ->
+                        comparison.firstSubmission.name.equals(nameA) &&
+                                comparison.secondSubmission.name.equals(nameB) ||
+                                comparison.firstSubmission.name.equals(nameB) &&
+                                        comparison.secondSubmission.name.equals(nameA))
+                .findFirst();
+    }
 }

jplag/src/test/java/jplag/OptionsTest.java

@@ -0,0 +1,5 @@
+package jplag;
+
+public class OptionsTest extends TestBase {
+    // TODO SH: There should be more tests testing with different combinations of JPlagOptions, e.g., base code
+}

jplag/src/test/java/jplag/TestBase.java

@@ -9,8 +9,12 @@ public abstract class TestBase {
 
     private static final String BASE_PATH = "src/test/resources/samples";
 
+    protected String getBasePath() {
+        return BASE_PATH;
+    }
+
     protected JPlagResult runJPlagWithExclusionFile(String testSampleName, String exclusionFileName) throws ExitException {
-        String blackList = String.format(BASE_PATH + "/%s/%s", testSampleName, exclusionFileName);
+        String blackList = String.format(getBasePath() + "/%s/%s", testSampleName, exclusionFileName);
         return runJPlagWithOptions(testSampleName, options -> options.setExclusionFileName(blackList));
     }
 
@@ -19,11 +23,10 @@ public abstract class TestBase {
     }
 
     protected JPlagResult runJPlagWithOptions(String testSampleName, Consumer<JPlagOptions> customization) throws ExitException {
-        JPlagOptions options = new JPlagOptions(String.format(BASE_PATH + "/%s", testSampleName), LanguageOption.JAVA_1_9);
+        JPlagOptions options = new JPlagOptions(String.format(getBasePath() + "/%s", testSampleName), LanguageOption.JAVA_1_9);
         options.setDebugParser(true);
         customization.accept(options);
         JPlag jplag = new JPlag(options);
         return jplag.run();
     }
-
 }

jplag/src/test/java/jplag/VolumeTest.java

@@ -0,0 +1,78 @@
+package jplag;
+
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * These tests are not intended to be used automatically but rather manually prior to releases.
+ * These tests shall be used with large data sets (e.g., 100+ submissions) in order to validate the core functionality.
+ */
+public class VolumeTest extends TestBase {
+
+    /**
+     * This path ensures that volume testing data is not in the git repository (with default configuration)
+     */
+    private static final String VOLUME_BASE_PATH = "../../volume";
+
+    @Override
+    protected String getBasePath() {
+        return VOLUME_BASE_PATH;
+    }
+
+    /**
+     * This test requires a folder "data" with submissions and a file named "matches_avg.csv" inside the volume folder.
+     * Accepts a derivation of 0.1% in the matching percentage
+     */
+    @Test
+    public void volumeComparisonTest() throws ExitException, IOException {
+
+        // Always succeed if not executed in an appropriate environment
+        if(!new File(this.getBasePath()).exists()) {
+            return;
+        }
+
+        var results = runJPlagWithOptions("data",
+                jPlagOptions -> jPlagOptions.setMaxNumberOfMatches(-1));
+
+        var csv = readCSVResults(String.format("%s/%s", this.getBasePath(), "matches_avg.csv"));
+
+        assertEquals(csv.size(), results.getAllComparisons().size());
+        System.out.println("Volume test size: " + csv.size());
+
+        results.getAllComparisons().forEach(result -> {
+            var key = result.firstSubmission.name + result.secondSubmission.name;
+
+            assertTrue(csv.containsKey(key));
+            assertEquals(csv.getOrDefault(key, -1f), result.percent(), 0.1f);
+        });
+
+    }
+
+    private Map<String, Float> readCSVResults(String filePathAndName) throws IOException {
+        List<String> lines = Files.readAllLines(Path.of(filePathAndName));
+        var results = new HashMap<String, Float>();
+
+        lines.forEach(line -> {
+            var entries = line.split(";");
+
+            if(entries.length != 4) {
+                throw new IllegalArgumentException(String.format("Illegal line: '%s'", line));
+            }
+
+            results.put(entries[1] + entries[2], Float.parseFloat((entries[3])));
+        });
+
+        return results;
+    }
+}

jplag/src/test/resources/samples/NoDuplicate/A/AlsoNoDuplicate.java

@@ -0,0 +1,30 @@
+/**
+ * This is code from the TagParser class of Jplag
+ */
+public class AlsoNoDuplicate {
+    public static String parse(String message, String[] params) {
+        String[] tokens = message.split("[{}]", -1);
+        String result = tokens[0];
+
+        for (int i = 1; i < tokens.length; i += 2)    // Go to next tag position
+        {
+            try {
+                int ind = tokens[i].indexOf('_');
+                String num = (ind == -1) ? tokens[i] : tokens[i].substring(0, ind);
+                result += params[Integer.parseInt(num) - 1];
+            } catch (Exception ex) {
+                if (ex instanceof NumberFormatException || ex instanceof IndexOutOfBoundsException) {
+                    ex.printStackTrace();
+                    result += "{ILLEGAL PARAMETER INDEX \"" + tokens[i] + "\"}";
+                } else {
+                    throw (RuntimeException) ex;
+                }
+            }
+            if (i + 1 < tokens.length) {
+                result += tokens[i + 1];
+            }
+        }
+
+        return result;
+    }
+}

jplag/src/test/resources/samples/NoDuplicate/B/NoDuplicate.java

@@ -0,0 +1,5 @@
+public class NoDuplicate {
+    public static void main(String[] args) {
+        System.out.println("Hello World!");
+    }
+}

jplag/src/test/resources/samples/NoDuplicate/C/AnotherClass.java

@@ -0,0 +1,30 @@
+/**
+ * This is so pointless, I hope nobody looks at test cases, e.g., in 25 years from now
+ */
+public class AnotherClass {
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    private String name;
+    private String id;
+
+    public AnotherClass(String name, String id) {
+        this.name = name;
+        this.id = id;
+    }
+
+}

jplag/src/test/resources/samples/NoDuplicate/C/CertainlyNoDuplicate.java

@@ -0,0 +1,27 @@
+import java.util.Locale;
+import java.util.MissingResourceException;
+import java.util.ResourceBundle;
+
+/**
+ * This is code from the Messages class of Jplag
+ */
+public class CertainlyNoDuplicate {
+
+    private final ResourceBundle resourceBundle;
+
+    /**
+     * @param countryTag may be "de", "en", "fr", "es", "pt" or "ptbr"
+     */
+    public CertainlyNoDuplicate(String countryTag) {
+        String bundleName = "jplag.messages";
+        resourceBundle = ResourceBundle.getBundle(bundleName, new Locale(countryTag));
+    }
+
+    public String getString(String key) {
+        try {
+            return resourceBundle.getString(key);
+        } catch (MissingResourceException e) {
+            return '!' + key + '!';
+        }
+    }
+}

jplag/src/test/resources/samples/PartialPlagiarism/A/GSTiling.java

@@ -0,0 +1,247 @@
+/*
+ * Diese Klasse implementiert den GSTiling Algorithmus
+ * Allerdings ist sie sehr spezifisch auf die Klassen "Structure", "Token",
+ * sowie "Matches" und "Match" ausgelegt
+ */
+public class GSTiling implements TokenConstants {
+	private Matches matches = new Matches();
+
+	/*
+	 * creating hashes.
+	 * The hash-code will be written in every Token for the next <hash_length> token.
+	 * (includes the Token itself)
+	 * Das Ganze laeuft in linearer Zeit.
+	 * condition: 1 < hashLength < 26   !!!
+	 */
+
+	public void create_hashes(Structure s, int hashLength, boolean makeTable) {
+		// Hier wird die obere Grenze der Hash-Laenge festgelegt.
+		// Sie ist bestimmt durch die Bitzahl des 'int' Datentyps und der Anzahl
+		// der Token.
+		if (hashLength<1) hashLength = 1;
+		hashLength = (hashLength<26 ? hashLength : 25);
+
+		if (s.size()<hashLength) return;
+
+		int modulo = ((1 << 6) - 1);   // Modulo 64!
+
+		int loops = s.size()-hashLength;
+		s.table = (makeTable ? new Table(3*loops) : null);
+		int hash = 0;
+		int i;
+		int hashedLength = 0;
+		for (i=0; i<hashLength; i++) {
+			hash = (2*hash) + (s.tokens[i].type & modulo);
+			hashedLength++;
+			if (s.tokens[i].marked)
+				hashedLength = 0;
+		}
+		int factor = (hashLength != 1 ? (2<<(hashLength-2)) : 1);
+
+		if (makeTable) {
+			for (i=0; i<loops; i++) {
+				if (hashedLength >= hashLength) {
+					s.tokens[i].hash = hash;
+					s.table.add(hash, i);   // add into hashtable
+				} else
+					s.tokens[i].hash = -1;
+				hash -= factor * (s.tokens[i].type & modulo);
+				hash = (2*hash) + (s.tokens[i+hashLength].type & modulo);
+				if (s.tokens[i+hashLength].marked)
+					hashedLength = 0;
+				else
+					hashedLength++;
+			}
+		} else {
+			for (i=0; i<loops; i++) {
+				s.tokens[i].hash = (hashedLength >= hashLength) ? hash : -1;
+				hash -= factor * (s.tokens[i].type & modulo);
+				hash = (2*hash) + (s.tokens[i+hashLength].type & modulo);
+				if (s.tokens[i+hashLength].marked)
+					hashedLength = 0;
+				else
+					hashedLength++;
+			}
+		}
+		s.hash_length = hashLength;
+	}
+
+	public final void compare(Submission subA, Submission subB) {
+		Submission A, B, tmp;
+		if (subA.struct.size() > subB.struct.size()) {
+			A = subB;  B = subA;
+		} else {
+			A = subB;  B = subA;
+		}
+		// if hashtable exists in first but not in second structure: flip around!
+		if (B.struct.table == null && A.struct.table != null) {
+			tmp = A;
+			A = B;
+			B = tmp;
+		}
+	}
+
+	// first parameter should contain the smaller sequence!!! 
+	private final void compare(Submission subA, Submission subB, int mml) {
+		Structure structA = subA.struct;
+		Structure structB = subB.struct;
+
+		// FILE_END used as pivot
+
+		// init
+		Token[] A = structA.tokens;
+		Token[] B = structB.tokens;
+		int lengthA = structA.size()-1;  // minus pivots!
+		int lengthB = structB.size()-1;  // minus pivots!
+
+
+		if(true) {
+			for(int i = 0; i <= lengthA; i++)
+				A[i].marked = A[i].type == FILE_END || A[i].type == SEPARATOR_TOKEN;
+
+			for(int i = 0; i <= lengthB; i++)
+				B[i].marked = B[i].type == FILE_END || B[i].type == SEPARATOR_TOKEN;
+		} else {
+			for(int i = 0; i <= lengthA; i++)
+				A[i].marked = A[i].type == FILE_END || A[i].type == SEPARATOR_TOKEN || A[i].basecode;
+
+			for(int i = 0; i <= lengthB; i++)
+				B[i].marked = B[i].type == FILE_END || B[i].type == SEPARATOR_TOKEN || B[i].basecode;
+		}
+
+
+		int maxmatch;
+		int[] elemsB;
+
+		do {
+			maxmatch = mml;
+			matches.clear();
+			for (int x = 0; x <= lengthA - maxmatch; x++) {
+				if (A[x].marked || A[x].hash == -1
+						|| (elemsB = structB.table.get(A[x].hash)) == null)
+					continue;
+inner:			for (int i = 1; i <= elemsB[0]; i++) { // elemsB[0] contains the length of the Array
+					int y = elemsB[i];
+					if (B[y].marked || maxmatch > lengthB - y) continue;
+
+					int j, hx, hy;
+					for (j = maxmatch - 1; j >= 0; j--) { //begins comparison from behind
+						if (A[hx = x + j].type != B[hy = y + j].type || A[hx].marked || B[hy].marked)
+							continue inner;
+					}
+
+					// expand match
+					j = maxmatch;
+					while(A[hx = x + j].type == B[hy = y + j].type && !A[hx].marked && !B[hy].marked)
+						j++;
+
+					if (j > maxmatch) {  // new biggest match? -> delete current smaller
+						matches.clear();
+						maxmatch = j;
+					}
+					matches.addMatch(x, y, j);  // add match
+				}
+			}
+			for (int i = matches.size() - 1; i >= 0; i--) {
+				int x = matches.matches[i].startA;  // begining of sequence A
+				int y = matches.matches[i].startB;  // begining of sequence B
+
+				//in order that "Match" will be newly build     (because reusing)
+				for (int j = matches.matches[i].length; j > 0; j--)
+					A[x++].marked = B[y++].marked = true;   // mark all Token!
+			}
+
+		} while (maxmatch != mml);
+	}
+
+	public final void compareWithBasecode(Submission subA, Submission subB) {
+		Submission A, B, tmp;
+		if (subA.struct.size() > subB.struct.size()) {
+			A = subB;  B = subA;
+		} else {
+			A = subB;  B = subA;
+		}
+		// if hashtable exists in first but not in second structure: flip around!
+		if (B.struct.table == null && A.struct.table != null) {
+			tmp = A;
+			A = B;
+			B = tmp;
+		}
+	}
+
+	private final void compareWithBasecode(Submission subA, Submission subB, int mml) {
+		Structure structA = subA.struct;
+		Structure structB = subB.struct;
+
+		// FILE_END used as pivot
+
+		// init
+		Token[] A = structA.tokens;
+		Token[] B = structB.tokens;
+		int lengthA = structA.size()-1;  // minus pivots!
+		int lengthB = structB.size()-1;  // minus pivots!
+
+
+		// Initialize
+		for(int i = 0; i <= lengthA; i++)
+			A[i].marked = A[i].type == FILE_END || A[i].type == SEPARATOR_TOKEN;
+
+		for(int i = 0; i <= lengthB; i++)
+			B[i].marked = B[i].type == FILE_END || B[i].type == SEPARATOR_TOKEN;
+
+
+
+
+		int maxmatch;
+		int[] elemsB;
+
+		do {
+			maxmatch = mml;
+			matches.clear();
+			for (int x = 0; x <= lengthA - maxmatch; x++) {
+				if (A[x].marked || A[x].hash == -1 || (elemsB = structB.table.get(A[x].hash)) == null)
+					continue;
+inner:			for (int i = 1; i <= elemsB[0]; i++) {// elemsB[0] contains the length of the Array
+					int y = elemsB[i];
+					if (B[y].marked || maxmatch > lengthB - y) continue;
+
+					int j,hx,hy;
+					for (j = maxmatch - 1; j >= 0; j--) { // begins comparison from behind
+						if (A[hx = x + j].type != B[hy = y + j].type || A[hx].marked || B[hy].marked)
+							continue inner;
+					}
+					// expand match
+					j = maxmatch;
+					while(A[hx = x + j].type == B[hy = y + j].type && !A[hx].marked && !B[hy].marked)
+						j++;
+
+					if (j != maxmatch) {  // new biggest match? -> delete current smaller
+						matches.clear();
+						maxmatch = j;
+					}
+					matches.addMatch(x, y, j);  // add match
+				}
+			}
+			for (int i = matches.size() - 1; i >= 0; i--) {
+				int x = matches.matches[i].startA;  // beginning in sequence A
+				int y = matches.matches[i].startB;  // beginning in sequence B
+
+				//in order that "Match" will be newly build     (because reusing)
+				for (int j = matches.matches[i].length; j > 0; j--){
+					A[x].marked = B[y].marked = true;   // mark all Token!
+					A[x].basecode = B[y].basecode= true;
+					x++; y++;
+				}
+			}
+		} while (maxmatch != mml);
+
+	}
+
+	public void resetBaseSubmission(Submission sub){
+		Structure tmpStruct = sub.struct;
+		Token[] tok = tmpStruct.tokens;
+		for (int z = 0; z < tmpStruct.size()-1;z++){
+			tok[z].basecode = false;
+		}
+	}
+}

jplag/src/test/resources/samples/PartialPlagiarism/A/Match.java

@@ -0,0 +1,52 @@
+public class Match {
+  public int startA;
+  public int startB;
+  public int length;
+
+  public Match(int startA, int startB, int length) {
+    this.startA = startA;
+    this.startB = startB;
+    this.length = length;
+  }
+  public Match() {}
+
+  public void set(int startA, int startB, int length) {
+    this.startA = startA;
+    this.startB = startB;
+    this.length = length;
+  }
+
+  public final boolean contains(int index, int sub) {
+    int start = (sub==0 ? startA : startB);
+    return (start <= index && index < (start+length));
+  }
+
+  public final boolean overlap(Match other) {
+    if (startA < other.startA) {
+      if ((other.startA - startA) < length) return true;
+    } else {
+      if ((startA - other.startA) < other.length) return true;
+    }
+    if (startB < other.startB) {
+      if ((other.startB - startB) < length) return true;
+    } else {
+      if ((startB - other.startB) < other.length) return true;
+    }
+    return false;
+  }
+
+  public final boolean overlap(int oStartA, int oStartB, int oLength) {
+    if (startA < oStartA) {
+      if ((oStartA - startA) < length) return true;
+    } else {
+      if ((startA - oStartA) < oLength) return true;
+    }
+    if (startB < oStartB) {
+      if ((oStartB - startB) < length) return true;
+    } else {
+      if ((startB - oStartB) < oLength) return true;
+    }
+    return false;
+  }
+}
+

jplag/src/test/resources/samples/PartialPlagiarism/A/Matches.java

@@ -0,0 +1,57 @@
+/* Minimal class that stores "Match" objects.
+ */
+public class Matches {
+  public Match[] matches;
+
+  private int anzahl;
+  private final int increment = 20;
+  
+  public Matches() {
+    matches = new Match[10];
+    for (int i=0; i<10; i++) matches[i] = new Match();
+    anzahl = 0;
+  }
+
+  public final int size() {
+    return anzahl;
+  }
+
+  public final void ensureCapacity(int minCapacity) {
+    int oldCapacity = matches.length;
+    if (minCapacity > oldCapacity) {
+      Match[] oldMatches = matches;
+      int newCapacity = (oldCapacity + increment);
+      if (newCapacity < minCapacity) {
+	newCapacity = minCapacity;
+      }
+      matches = new Match[newCapacity];
+      System.arraycopy(oldMatches, 0, matches, 0, oldCapacity);
+      for (int i=oldCapacity; i<newCapacity; i++)
+	matches[i] = new Match();
+    }
+  }
+  /*
+  public final void addMatch(Match match) {
+    for (int i=0; i<anzahl; i++)
+      if (match.overlap(matches[i])) return;  // do not allow overlaps
+    ensureCapacity(anzahl + 1);
+    matches[anzahl++] = match;
+  }
+  */
+  public final void addMatch(int startA, int startB, int length) {
+    for (int i=anzahl-1; i>=0; i--) { // starting at the end is better(?)
+      if (matches[i].overlap(startA,startB,length)) return;
+                                      // no overlaps!
+    }
+    ensureCapacity(anzahl + 1);
+    //if (matches[anzahl] != null)  // object recycling...
+    matches[anzahl].set(startA,startB,length);
+    //else
+    //matches[anzahl ] = new Match(startA,startB,length);
+    anzahl++;
+  }
+
+  public final void clear() {
+    anzahl=0;
+  }
+}

jplag/src/test/resources/samples/PartialPlagiarism/A/Structure.java

@@ -0,0 +1,118 @@
+import java.io.*;
+
+/** The tokenlist */
+public class Structure implements TokenConstants {
+	public Token[] tokens = new Token[0];
+	Table table = null;
+	int hash_length = -1;
+
+	int files; // number of END_FILE tokens
+	private int anzahl;
+
+	//private final int increment = 300;
+
+	public Structure() {
+		tokens = new Token[400];
+		files = anzahl = 0;
+	}
+
+	public final int size() {
+		return anzahl;
+	}
+
+	public final void ensureCapacity(int minCapacity) {
+		int oldCapacity = tokens.length;
+		if (minCapacity > oldCapacity) {
+			Token[] oldTokens = tokens;
+			int newCapacity = (2 * oldCapacity);
+			if (newCapacity < minCapacity) {
+				newCapacity = minCapacity;
+			}
+			tokens = new Token[newCapacity];
+			System.arraycopy(oldTokens, 0, tokens, 0, anzahl);
+		}
+	}
+
+	public final void addToken(Token token) {
+		ensureCapacity(anzahl + 1);
+		if (anzahl > 0 && tokens[anzahl - 1].file.equals(token.file))
+			token.file = tokens[anzahl - 1].file; // To save memory ...
+		if ((anzahl > 0) && (token.getLine() < tokens[anzahl - 1].getLine()) && (token.file.equals(tokens[anzahl - 1].file)))
+			token.setLine(tokens[anzahl - 1].getLine());
+		// just to make sure
+
+		tokens[anzahl++] = token;
+		if (token.type == FILE_END)
+			files++;
+	}
+
+	public final String toString() {
+		StringBuffer buf = new StringBuffer();
+
+		try {
+			for (int i = 0; i < anzahl; i++) {
+				String s = tokens[i].toString();
+				buf.append(i);
+				buf.append("\t");
+				buf.append(s);
+				if (i < anzahl - 1) {
+					buf.append("\n");
+				}
+			}
+		} catch (OutOfMemoryError e) {
+			return "Tokenlist to large for output: " + (anzahl) + " Tokens";
+		}
+		return buf.toString();
+	}
+
+	public void save(File file) {
+
+		try {
+			ObjectOutputStream p = new ObjectOutputStream(/* new GZIPOutputStream */(new FileOutputStream(file)));
+
+			p.writeInt(anzahl);
+			p.writeInt(hash_length);
+			p.writeInt(files);
+
+			for (int i = 0; i < anzahl; i++)
+				p.writeObject(tokens[i]);
+			p.flush();
+			p.close();
+		} catch (IOException e) {
+			System.out.println("Error writing file: " + file.toString());
+		}
+	}
+
+	/* returns "true" when successful */
+	public boolean load(File file) {
+		try {
+			ObjectInputStream p = new ObjectInputStream(/* new GZIPInputStream */(new FileInputStream(file)));
+
+			int newAnzahl = p.readInt();
+			hash_length = p.readInt();
+			files = p.readInt();
+			ensureCapacity(newAnzahl);
+			anzahl = newAnzahl;
+			for (int i = 0; i < anzahl; i++) {
+				tokens[i] = (Token) p.readObject();
+				// special case for text tokens:
+				// 	if (tokens[i] instanceof jplag.text.TextToken) {
+				// 	  jplag.text.TextToken token = (jplag.text.TextToken)tokens[i];
+				// 	  jplag.text.TextToken.put(token.getText(), token.type);
+				// 	}
+			}
+			p.close();
+			table = null;
+		} catch (FileNotFoundException e) {
+			System.out.println("File not found: " + file.toString());
+			return false;
+		} catch (IOException e) {
+			System.out.println("Error reading file: " + file.toString() + " (" + e + ")");
+			return false;
+		} catch (ClassNotFoundException e) {
+			System.out.println("Class not found in file: " + file.toString());
+			return false;
+		}
+		return true;
+	}
+}

jplag/src/test/resources/samples/PartialPlagiarism/A/Submission.java

@@ -0,0 +1,218 @@
+import java.io.*;
+import java.net.URL;
+import java.text.DecimalFormat;
+import java.util.Vector;
+
+/*
+ * Everything about a single submission is stored in this object. (directory,
+ * files, ...)
+ */
+public class Submission implements Comparable<Submission> {
+	public String name;
+
+	private boolean readSubDirs;
+
+	public File dir;
+
+	public String[] files = null; // = new String[0];
+
+	public Structure struct;
+
+	public int structSize = 0;
+
+	// public long structMem;
+	boolean exact_match = false; // used for fallback
+
+	public boolean errors = false;
+
+	public DecimalFormat format = new DecimalFormat("0000");
+
+	public Submission(String name, File dir, boolean readSubDirs) {
+		this.dir = dir;
+		this.name = name;
+		this.readSubDirs = readSubDirs;
+		try {
+			lookupDir(dir, "");
+		} catch (Throwable b) {
+		}
+	}
+
+	public Submission(String name, File dir) {
+		this.dir = dir;
+		this.name = name;
+		this.readSubDirs = false;
+
+		files = new String[1];
+		files[0] = name;
+
+	}
+
+	// recursively read in all the files
+	private void lookupDir(File dir, String subDir) throws Throwable {
+		File aktDir = new File(dir, subDir);
+		if (!aktDir.isDirectory())
+			return;
+		if (readSubDirs) {
+			String[] dirs = aktDir.list();
+			if (!subDir.equals(""))
+				for (int i = 0; i < dirs.length; i++)
+					lookupDir(dir, subDir + File.separator + dirs[i]);
+			else
+				for (int i = 0; i < dirs.length; i++)
+					lookupDir(dir, dirs[i]);
+		}
+		String[] newFiles = aktDir.list(new FilenameFilter() {
+			public boolean accept(File dir, String name) {
+					return false;
+			}
+		});
+		if (files != null) {
+			String[] oldFiles = files;
+			files = new String[oldFiles.length + newFiles.length];
+			if (subDir != "")
+				for (int i = 0; i < newFiles.length; i++)
+					files[i] = subDir + File.separator + newFiles[i];
+			else
+				System.arraycopy(newFiles, 0, files, 0, newFiles.length);
+			System.arraycopy(oldFiles, 0, files, newFiles.length, oldFiles.length);
+		} else {
+			if (subDir != "") {
+				files = new String[newFiles.length];
+				for (int i = 0; i < newFiles.length; i++)
+					files[i] = subDir + File.separator + newFiles[i];
+			} else
+				files = newFiles;
+		}
+	}
+
+	/* parse all the files... */
+	public boolean parse(){
+		return false;
+	}
+
+	/*
+	 * This method is used to copy files that can not be parsed to a special
+	 * folder: jplag/errors/java old_java scheme cpp /001/(...files...)
+	 * /002/(...files...)
+	 */
+	private void copySubmission() {
+		File errorDir = null;
+		try {
+			URL url = Submission.class.getResource("errors");
+			errorDir = new File(url.getFile());
+		} catch (NullPointerException e) {
+			return;
+		}
+
+		if (!errorDir.exists())
+			errorDir.mkdir();
+		int i = 0;
+		File destDir;
+		while ((destDir = new File(errorDir, format.format(i))).exists())
+			i++;
+		destDir.mkdir();
+		for (i = 0; i < files.length; i++)
+			copyFile(new File(dir, files[i]), new File(destDir, files[i]));
+	}
+
+	/* Physical copy. :-) */
+	private void copyFile(File in, File out) {
+		byte[] buffer = new byte[10000];
+		try {
+			FileInputStream dis = new FileInputStream(in);
+			FileOutputStream dos = new FileOutputStream(out);
+			int count;
+			do {
+				count = dis.read(buffer);
+				if (count != -1)
+					dos.write(buffer, 0, count);
+			} while (count != -1);
+			dis.close();
+			dos.close();
+		} catch (IOException e) {
+
+		}
+	}
+
+	public int size() {
+		if (struct != null)
+			return structSize = struct.size();
+		return structSize;
+	}
+
+	/*
+	 * Used by the "Report" class. All source files are returned as an array of
+	 * an array of strings.
+	 */
+	public String[][] readFiles(String[] files) {
+		String[][] result = new String[files.length][];
+		String help;
+
+		Vector<String> text = new Vector<String>();
+		for (int i = 0; i < files.length; i++) {
+			text.removeAllElements();
+			try {
+				/* file encoding = "UTF-8" */
+				FileInputStream fileInputStream = new FileInputStream(new File(dir, files[i]));
+				InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8");
+				BufferedReader in = new BufferedReader(inputStreamReader);
+				while ((help = in.readLine()) != null) {
+					help = help.replaceAll("&", "&amp;");
+					help = help.replaceAll("<", "&lt;");
+					help = help.replaceAll(">", "&gt;");
+					help = help.replaceAll("\"", "&quot;");
+					text.addElement(help);
+				}
+				in.close();
+				inputStreamReader.close();
+				fileInputStream.close();
+			} catch (FileNotFoundException e) {
+				System.out.println("File not found: " + ((new File(dir, files[i])).toString()));
+			} catch (IOException e) {
+
+			}
+			result[i] = new String[text.size()];
+			text.copyInto(result[i]);
+		}
+		return result;
+	}
+
+	/*
+	 * Used by the "Report" class. All source files are returned as an array of
+	 * an array of chars.
+	 */
+	public char[][] readFilesChar(String[] files) {
+		char[][] result = new char[files.length][];
+
+		for (int i = 0; i < files.length; i++) {
+			try {
+				File file = new File(dir, files[i]);
+				int size = (int) file.length();
+				char[] buffer = new char[size];
+
+				FileReader fis = new FileReader(file);
+
+				if (size != fis.read(buffer)) {
+					System.out.println("Not right size read from the file, " + "but I will still continue...");
+				}
+
+				result[i] = buffer;
+				fis.close();
+			} catch (FileNotFoundException e) {
+				// TODO: Should an ExitException be thrown here?
+				System.out.println("File not found: " + ((new File(dir, files[i])).toString()));
+			} catch (IOException e) {
+
+			}
+		}
+		return result;
+	}
+
+	public int compareTo(Submission other) {
+		return name.compareTo(other.name);
+	}
+
+	public String toString() {
+		return name;
+	}
+}

jplag/src/test/resources/samples/PartialPlagiarism/A/Token.java

@@ -0,0 +1,47 @@
+import java.io.Serializable;
+
+abstract public class Token implements TokenConstants, Serializable {
+	private static final long serialVersionUID = 862935679966383302L;
+	public int type;
+	public String file;
+	
+	protected boolean marked;
+	protected boolean basecode = false;
+	protected int hash = -1;// hash-value. set and used by main algorithm (GSTiling)
+	
+	public Token(int type, String file, int line) {
+		this(type, file, line, -1, -1);
+	}
+	
+	public Token(int type, String file, int line, int column, int length) {
+		this.type = type;
+		this.file = file;
+		setLine(line>0 ? line : 1);
+		setColumn(column);
+		setLength(length);
+	}
+	
+	// abstract members
+	abstract public int getLine();
+	abstract public int getColumn();
+	abstract public int getLength();
+	protected void setLine(int line) {}
+	protected void setColumn(int line) {}
+	protected void setLength(int line) {}
+	
+	// this is made to distinguish the character front end.
+	// maybe other front ends can use it too?
+	protected int getIndex() { return -1; }
+	
+	public static String type2string(int type) {
+		return "<abstract>";
+	}
+	
+	public String toString() {
+		return type2string(type);
+	}
+	
+	public static int numberOfTokens() { 
+		return 1;
+	} 
+}

jplag/src/test/resources/samples/PartialPlagiarism/A/TokenConstants.java

@@ -0,0 +1,8 @@
+public interface TokenConstants {
+  final static int FILE_END = 0;
+	/**
+	 * Used to optionally separate methods from each other with an always marked
+	 * token
+	 */
+  final static int SEPARATOR_TOKEN = 1;
+}

jplag/src/test/resources/samples/PartialPlagiarism/B/Token.java

@@ -0,0 +1,47 @@
+import java.io.Serializable;
+
+abstract public class Token implements TokenConstants, Serializable {
+	private static final long serialVersionUID = 862935679966383302L;
+	public int type;
+	public String file;
+	
+	protected boolean marked;
+	protected boolean basecode = false;
+	protected int hash = -1;// hash-value. set and used by main algorithm (GSTiling)
+	
+	public Token(int type, String file, int line) {
+		this(type, file, line, -1, -1);
+	}
+	
+	public Token(int type, String file, int line, int column, int length) {
+		this.type = type;
+		this.file = file;
+		setLine(line>0 ? line : 1);
+		setColumn(column);
+		setLength(length);
+	}
+	
+	// abstract members
+	abstract public int getLine();
+	abstract public int getColumn();
+	abstract public int getLength();
+	protected void setLine(int line) {}
+	protected void setColumn(int line) {}
+	protected void setLength(int line) {}
+	
+	// this is made to distinguish the character front end.
+	// maybe other front ends can use it too?
+	protected int getIndex() { return -1; }
+	
+	public static String type2string(int type) {
+		return "<abstract>";
+	}
+	
+	public String toString() {
+		return type2string(type);
+	}
+	
+	public static int numberOfTokens() { 
+		return 1;
+	} 
+}

jplag/src/test/resources/samples/PartialPlagiarism/B/TokenConstants.java

@@ -0,0 +1,8 @@
+public interface TokenConstants {
+  final static int FILE_END = 0;
+	/**
+	 * Used to optionally separate methods from each other with an always marked
+	 * token
+	 */
+  final static int SEPARATOR_TOKEN = 1;
+}

jplag/src/test/resources/samples/PartialPlagiarism/C/GSTiling.java

@@ -0,0 +1,247 @@
+/*
+ * Diese Klasse implementiert den GSTiling Algorithmus
+ * Allerdings ist sie sehr spezifisch auf die Klassen "Structure", "Token",
+ * sowie "Matches" und "Match" ausgelegt
+ */
+public class GSTiling implements TokenConstants {
+	private Matches matches = new Matches();
+
+	/*
+	 * creating hashes.
+	 * The hash-code will be written in every Token for the next <hash_length> token.
+	 * (includes the Token itself)
+	 * Das Ganze laeuft in linearer Zeit.
+	 * condition: 1 < hashLength < 26   !!!
+	 */
+
+	public void create_hashes(Structure s, int hashLength, boolean makeTable) {
+		// Hier wird die obere Grenze der Hash-Laenge festgelegt.
+		// Sie ist bestimmt durch die Bitzahl des 'int' Datentyps und der Anzahl
+		// der Token.
+		if (hashLength<1) hashLength = 1;
+		hashLength = (hashLength<26 ? hashLength : 25);
+
+		if (s.size()<hashLength) return;
+
+		int modulo = ((1 << 6) - 1);   // Modulo 64!
+
+		int loops = s.size()-hashLength;
+		s.table = (makeTable ? new Table(3*loops) : null);
+		int hash = 0;
+		int i;
+		int hashedLength = 0;
+		for (i=0; i<hashLength; i++) {
+			hash = (2*hash) + (s.tokens[i].type & modulo);
+			hashedLength++;
+			if (s.tokens[i].marked)
+				hashedLength = 0;
+		}
+		int factor = (hashLength != 1 ? (2<<(hashLength-2)) : 1);
+
+		if (makeTable) {
+			for (i=0; i<loops; i++) {
+				if (hashedLength >= hashLength) {
+					s.tokens[i].hash = hash;
+					s.table.add(hash, i);   // add into hashtable
+				} else
+					s.tokens[i].hash = -1;
+				hash -= factor * (s.tokens[i].type & modulo);
+				hash = (2*hash) + (s.tokens[i+hashLength].type & modulo);
+				if (s.tokens[i+hashLength].marked)
+					hashedLength = 0;
+				else
+					hashedLength++;
+			}
+		} else {
+			for (i=0; i<loops; i++) {
+				s.tokens[i].hash = (hashedLength >= hashLength) ? hash : -1;
+				hash -= factor * (s.tokens[i].type & modulo);
+				hash = (2*hash) + (s.tokens[i+hashLength].type & modulo);
+				if (s.tokens[i+hashLength].marked)
+					hashedLength = 0;
+				else
+					hashedLength++;
+			}
+		}
+		s.hash_length = hashLength;
+	}
+
+	public final void compare(Submission subA, Submission subB) {
+		Submission A, B, tmp;
+		if (subA.struct.size() > subB.struct.size()) {
+			A = subB;  B = subA;
+		} else {
+			A = subB;  B = subA;
+		}
+		// if hashtable exists in first but not in second structure: flip around!
+		if (B.struct.table == null && A.struct.table != null) {
+			tmp = A;
+			A = B;
+			B = tmp;
+		}
+	}
+
+	// first parameter should contain the smaller sequence!!! 
+	private final void compare(Submission subA, Submission subB, int mml) {
+		Structure structA = subA.struct;
+		Structure structB = subB.struct;
+
+		// FILE_END used as pivot
+
+		// init
+		Token[] A = structA.tokens;
+		Token[] B = structB.tokens;
+		int lengthA = structA.size()-1;  // minus pivots!
+		int lengthB = structB.size()-1;  // minus pivots!
+
+
+		if(true) {
+			for(int i = 0; i <= lengthA; i++)
+				A[i].marked = A[i].type == FILE_END || A[i].type == SEPARATOR_TOKEN;
+
+			for(int i = 0; i <= lengthB; i++)
+				B[i].marked = B[i].type == FILE_END || B[i].type == SEPARATOR_TOKEN;
+		} else {
+			for(int i = 0; i <= lengthA; i++)
+				A[i].marked = A[i].type == FILE_END || A[i].type == SEPARATOR_TOKEN || A[i].basecode;
+
+			for(int i = 0; i <= lengthB; i++)
+				B[i].marked = B[i].type == FILE_END || B[i].type == SEPARATOR_TOKEN || B[i].basecode;
+		}
+
+
+		int maxmatch;
+		int[] elemsB;
+
+		do {
+			maxmatch = mml;
+			matches.clear();
+			for (int x = 0; x <= lengthA - maxmatch; x++) {
+				if (A[x].marked || A[x].hash == -1
+						|| (elemsB = structB.table.get(A[x].hash)) == null)
+					continue;
+inner:			for (int i = 1; i <= elemsB[0]; i++) { // elemsB[0] contains the length of the Array
+					int y = elemsB[i];
+					if (B[y].marked || maxmatch > lengthB - y) continue;
+
+					int j, hx, hy;
+					for (j = maxmatch - 1; j >= 0; j--) { //begins comparison from behind
+						if (A[hx = x + j].type != B[hy = y + j].type || A[hx].marked || B[hy].marked)
+							continue inner;
+					}
+
+					// expand match
+					j = maxmatch;
+					while(A[hx = x + j].type == B[hy = y + j].type && !A[hx].marked && !B[hy].marked)
+						j++;
+
+					if (j > maxmatch) {  // new biggest match? -> delete current smaller
+						matches.clear();
+						maxmatch = j;
+					}
+					matches.addMatch(x, y, j);  // add match
+				}
+			}
+			for (int i = matches.size() - 1; i >= 0; i--) {
+				int x = matches.matches[i].startA;  // begining of sequence A
+				int y = matches.matches[i].startB;  // begining of sequence B
+
+				//in order that "Match" will be newly build     (because reusing)
+				for (int j = matches.matches[i].length; j > 0; j--)
+					A[x++].marked = B[y++].marked = true;   // mark all Token!
+			}
+
+		} while (maxmatch != mml);
+	}
+
+	public final void compareWithBasecode(Submission subA, Submission subB) {
+		Submission A, B, tmp;
+		if (subA.struct.size() > subB.struct.size()) {
+			A = subB;  B = subA;
+		} else {
+			A = subB;  B = subA;
+		}
+		// if hashtable exists in first but not in second structure: flip around!
+		if (B.struct.table == null && A.struct.table != null) {
+			tmp = A;
+			A = B;
+			B = tmp;
+		}
+	}
+
+	private final void compareWithBasecode(Submission subA, Submission subB, int mml) {
+		Structure structA = subA.struct;
+		Structure structB = subB.struct;
+
+		// FILE_END used as pivot
+
+		// init
+		Token[] A = structA.tokens;
+		Token[] B = structB.tokens;
+		int lengthA = structA.size()-1;  // minus pivots!
+		int lengthB = structB.size()-1;  // minus pivots!
+
+
+		// Initialize
+		for(int i = 0; i <= lengthA; i++)
+			A[i].marked = A[i].type == FILE_END || A[i].type == SEPARATOR_TOKEN;
+
+		for(int i = 0; i <= lengthB; i++)
+			B[i].marked = B[i].type == FILE_END || B[i].type == SEPARATOR_TOKEN;
+
+
+
+
+		int maxmatch;
+		int[] elemsB;
+
+		do {
+			maxmatch = mml;
+			matches.clear();
+			for (int x = 0; x <= lengthA - maxmatch; x++) {
+				if (A[x].marked || A[x].hash == -1 || (elemsB = structB.table.get(A[x].hash)) == null)
+					continue;
+inner:			for (int i = 1; i <= elemsB[0]; i++) {// elemsB[0] contains the length of the Array
+					int y = elemsB[i];
+					if (B[y].marked || maxmatch > lengthB - y) continue;
+
+					int j,hx,hy;
+					for (j = maxmatch - 1; j >= 0; j--) { // begins comparison from behind
+						if (A[hx = x + j].type != B[hy = y + j].type || A[hx].marked || B[hy].marked)
+							continue inner;
+					}
+					// expand match
+					j = maxmatch;
+					while(A[hx = x + j].type == B[hy = y + j].type && !A[hx].marked && !B[hy].marked)
+						j++;
+
+					if (j != maxmatch) {  // new biggest match? -> delete current smaller
+						matches.clear();
+						maxmatch = j;
+					}
+					matches.addMatch(x, y, j);  // add match
+				}
+			}
+			for (int i = matches.size() - 1; i >= 0; i--) {
+				int x = matches.matches[i].startA;  // beginning in sequence A
+				int y = matches.matches[i].startB;  // beginning in sequence B
+
+				//in order that "Match" will be newly build     (because reusing)
+				for (int j = matches.matches[i].length; j > 0; j--){
+					A[x].marked = B[y].marked = true;   // mark all Token!
+					A[x].basecode = B[y].basecode= true;
+					x++; y++;
+				}
+			}
+		} while (maxmatch != mml);
+
+	}
+
+	public void resetBaseSubmission(Submission sub){
+		Structure tmpStruct = sub.struct;
+		Token[] tok = tmpStruct.tokens;
+		for (int z = 0; z < tmpStruct.size()-1;z++){
+			tok[z].basecode = false;
+		}
+	}
+}

jplag/src/test/resources/samples/PartialPlagiarism/C/Match.java

@@ -0,0 +1,52 @@
+public class Match {
+  public int startA;
+  public int startB;
+  public int length;
+
+  public Match(int startA, int startB, int length) {
+    this.startA = startA;
+    this.startB = startB;
+    this.length = length;
+  }
+  public Match() {}
+
+  public void set(int startA, int startB, int length) {
+    this.startA = startA;
+    this.startB = startB;
+    this.length = length;
+  }
+
+  public final boolean contains(int index, int sub) {
+    int start = (sub==0 ? startA : startB);
+    return (start <= index && index < (start+length));
+  }
+
+  public final boolean overlap(Match other) {
+    if (startA < other.startA) {
+      if ((other.startA - startA) < length) return true;
+    } else {
+      if ((startA - other.startA) < other.length) return true;
+    }
+    if (startB < other.startB) {
+      if ((other.startB - startB) < length) return true;
+    } else {
+      if ((startB - other.startB) < other.length) return true;
+    }
+    return false;
+  }
+
+  public final boolean overlap(int oStartA, int oStartB, int oLength) {
+    if (startA < oStartA) {
+      if ((oStartA - startA) < length) return true;
+    } else {
+      if ((startA - oStartA) < oLength) return true;
+    }
+    if (startB < oStartB) {
+      if ((oStartB - startB) < length) return true;
+    } else {
+      if ((startB - oStartB) < oLength) return true;
+    }
+    return false;
+  }
+}
+

jplag/src/test/resources/samples/PartialPlagiarism/C/Matches.java

@@ -0,0 +1,57 @@
+/* Minimal class that stores "Match" objects.
+ */
+public class Matches {
+  public Match[] matches;
+
+  private int anzahl;
+  private final int increment = 20;
+  
+  public Matches() {
+    matches = new Match[10];
+    for (int i=0; i<10; i++) matches[i] = new Match();
+    anzahl = 0;
+  }
+
+  public final int size() {
+    return anzahl;
+  }
+
+  public final void ensureCapacity(int minCapacity) {
+    int oldCapacity = matches.length;
+    if (minCapacity > oldCapacity) {
+      Match[] oldMatches = matches;
+      int newCapacity = (oldCapacity + increment);
+      if (newCapacity < minCapacity) {
+	newCapacity = minCapacity;
+      }
+      matches = new Match[newCapacity];
+      System.arraycopy(oldMatches, 0, matches, 0, oldCapacity);
+      for (int i=oldCapacity; i<newCapacity; i++)
+	matches[i] = new Match();
+    }
+  }
+  /*
+  public final void addMatch(Match match) {
+    for (int i=0; i<anzahl; i++)
+      if (match.overlap(matches[i])) return;  // do not allow overlaps
+    ensureCapacity(anzahl + 1);
+    matches[anzahl++] = match;
+  }
+  */
+  public final void addMatch(int startA, int startB, int length) {
+    for (int i=anzahl-1; i>=0; i--) { // starting at the end is better(?)
+      if (matches[i].overlap(startA,startB,length)) return;
+                                      // no overlaps!
+    }
+    ensureCapacity(anzahl + 1);
+    //if (matches[anzahl] != null)  // object recycling...
+    matches[anzahl].set(startA,startB,length);
+    //else
+    //matches[anzahl ] = new Match(startA,startB,length);
+    anzahl++;
+  }
+
+  public final void clear() {
+    anzahl=0;
+  }
+}

jplag/src/test/resources/samples/PartialPlagiarism/C/Structure.java

@@ -0,0 +1,118 @@
+import java.io.*;
+
+/** The tokenlist */
+public class Structure implements TokenConstants {
+	public Token[] tokens = new Token[0];
+	Table table = null;
+	int hash_length = -1;
+
+	int files; // number of END_FILE tokens
+	private int anzahl;
+
+	//private final int increment = 300;
+
+	public Structure() {
+		tokens = new Token[400];
+		files = anzahl = 0;
+	}
+
+	public final int size() {
+		return anzahl;
+	}
+
+	public final void ensureCapacity(int minCapacity) {
+		int oldCapacity = tokens.length;
+		if (minCapacity > oldCapacity) {
+			Token[] oldTokens = tokens;
+			int newCapacity = (2 * oldCapacity);
+			if (newCapacity < minCapacity) {
+				newCapacity = minCapacity;
+			}
+			tokens = new Token[newCapacity];
+			System.arraycopy(oldTokens, 0, tokens, 0, anzahl);
+		}
+	}
+
+	public final void addToken(Token token) {
+		ensureCapacity(anzahl + 1);
+		if (anzahl > 0 && tokens[anzahl - 1].file.equals(token.file))
+			token.file = tokens[anzahl - 1].file; // To save memory ...
+		if ((anzahl > 0) && (token.getLine() < tokens[anzahl - 1].getLine()) && (token.file.equals(tokens[anzahl - 1].file)))
+			token.setLine(tokens[anzahl - 1].getLine());
+		// just to make sure
+
+		tokens[anzahl++] = token;
+		if (token.type == FILE_END)
+			files++;
+	}
+
+	public final String toString() {
+		StringBuffer buf = new StringBuffer();
+
+		try {
+			for (int i = 0; i < anzahl; i++) {
+				String s = tokens[i].toString();
+				buf.append(i);
+				buf.append("\t");
+				buf.append(s);
+				if (i < anzahl - 1) {
+					buf.append("\n");
+				}
+			}
+		} catch (OutOfMemoryError e) {
+			return "Tokenlist to large for output: " + (anzahl) + " Tokens";
+		}
+		return buf.toString();
+	}
+
+	public void save(File file) {
+
+		try {
+			ObjectOutputStream p = new ObjectOutputStream(/* new GZIPOutputStream */(new FileOutputStream(file)));
+
+			p.writeInt(anzahl);
+			p.writeInt(hash_length);
+			p.writeInt(files);
+
+			for (int i = 0; i < anzahl; i++)
+				p.writeObject(tokens[i]);
+			p.flush();
+			p.close();
+		} catch (IOException e) {
+			System.out.println("Error writing file: " + file.toString());
+		}
+	}
+
+	/* returns "true" when successful */
+	public boolean load(File file) {
+		try {
+			ObjectInputStream p = new ObjectInputStream(/* new GZIPInputStream */(new FileInputStream(file)));
+
+			int newAnzahl = p.readInt();
+			hash_length = p.readInt();
+			files = p.readInt();
+			ensureCapacity(newAnzahl);
+			anzahl = newAnzahl;
+			for (int i = 0; i < anzahl; i++) {
+				tokens[i] = (Token) p.readObject();
+				// special case for text tokens:
+				// 	if (tokens[i] instanceof jplag.text.TextToken) {
+				// 	  jplag.text.TextToken token = (jplag.text.TextToken)tokens[i];
+				// 	  jplag.text.TextToken.put(token.getText(), token.type);
+				// 	}
+			}
+			p.close();
+			table = null;
+		} catch (FileNotFoundException e) {
+			System.out.println("File not found: " + file.toString());
+			return false;
+		} catch (IOException e) {
+			System.out.println("Error reading file: " + file.toString() + " (" + e + ")");
+			return false;
+		} catch (ClassNotFoundException e) {
+			System.out.println("Class not found in file: " + file.toString());
+			return false;
+		}
+		return true;
+	}
+}

jplag/src/test/resources/samples/PartialPlagiarism/C/Submission.java

@@ -0,0 +1,218 @@
+import java.io.*;
+import java.net.URL;
+import java.text.DecimalFormat;
+import java.util.Vector;
+
+/*
+ * Everything about a single submission is stored in this object. (directory,
+ * files, ...)
+ */
+public class Submission implements Comparable<Submission> {
+	public String name;
+
+	private boolean readSubDirs;
+
+	public File dir;
+
+	public String[] files = null; // = new String[0];
+
+	public Structure struct;
+
+	public int structSize = 0;
+
+	// public long structMem;
+	boolean exact_match = false; // used for fallback
+
+	public boolean errors = false;
+
+	public DecimalFormat format = new DecimalFormat("0000");
+
+	public Submission(String name, File dir, boolean readSubDirs) {
+		this.dir = dir;
+		this.name = name;
+		this.readSubDirs = readSubDirs;
+		try {
+			lookupDir(dir, "");
+		} catch (Throwable b) {
+		}
+	}
+
+	public Submission(String name, File dir) {
+		this.dir = dir;
+		this.name = name;
+		this.readSubDirs = false;
+
+		files = new String[1];
+		files[0] = name;
+
+	}
+
+	// recursively read in all the files
+	private void lookupDir(File dir, String subDir) throws Throwable {
+		File aktDir = new File(dir, subDir);
+		if (!aktDir.isDirectory())
+			return;
+		if (readSubDirs) {
+			String[] dirs = aktDir.list();
+			if (!subDir.equals(""))
+				for (int i = 0; i < dirs.length; i++)
+					lookupDir(dir, subDir + File.separator + dirs[i]);
+			else
+				for (int i = 0; i < dirs.length; i++)
+					lookupDir(dir, dirs[i]);
+		}
+		String[] newFiles = aktDir.list(new FilenameFilter() {
+			public boolean accept(File dir, String name) {
+					return false;
+			}
+		});
+		if (files != null) {
+			String[] oldFiles = files;
+			files = new String[oldFiles.length + newFiles.length];
+			if (subDir != "")
+				for (int i = 0; i < newFiles.length; i++)
+					files[i] = subDir + File.separator + newFiles[i];
+			else
+				System.arraycopy(newFiles, 0, files, 0, newFiles.length);
+			System.arraycopy(oldFiles, 0, files, newFiles.length, oldFiles.length);
+		} else {
+			if (subDir != "") {
+				files = new String[newFiles.length];
+				for (int i = 0; i < newFiles.length; i++)
+					files[i] = subDir + File.separator + newFiles[i];
+			} else
+				files = newFiles;
+		}
+	}
+
+	/* parse all the files... */
+	public boolean parse(){
+		return false;
+	}
+
+	/*
+	 * This method is used to copy files that can not be parsed to a special
+	 * folder: jplag/errors/java old_java scheme cpp /001/(...files...)
+	 * /002/(...files...)
+	 */
+	private void copySubmission() {
+		File errorDir = null;
+		try {
+			URL url = Submission.class.getResource("errors");
+			errorDir = new File(url.getFile());
+		} catch (NullPointerException e) {
+			return;
+		}
+
+		if (!errorDir.exists())
+			errorDir.mkdir();
+		int i = 0;
+		File destDir;
+		while ((destDir = new File(errorDir, format.format(i))).exists())
+			i++;
+		destDir.mkdir();
+		for (i = 0; i < files.length; i++)
+			copyFile(new File(dir, files[i]), new File(destDir, files[i]));
+	}
+
+	/* Physical copy. :-) */
+	private void copyFile(File in, File out) {
+		byte[] buffer = new byte[10000];
+		try {
+			FileInputStream dis = new FileInputStream(in);
+			FileOutputStream dos = new FileOutputStream(out);
+			int count;
+			do {
+				count = dis.read(buffer);
+				if (count != -1)
+					dos.write(buffer, 0, count);
+			} while (count != -1);
+			dis.close();
+			dos.close();
+		} catch (IOException e) {
+
+		}
+	}
+
+	public int size() {
+		if (struct != null)
+			return structSize = struct.size();
+		return structSize;
+	}
+
+	/*
+	 * Used by the "Report" class. All source files are returned as an array of
+	 * an array of strings.
+	 */
+	public String[][] readFiles(String[] files) {
+		String[][] result = new String[files.length][];
+		String help;
+
+		Vector<String> text = new Vector<String>();
+		for (int i = 0; i < files.length; i++) {
+			text.removeAllElements();
+			try {
+				/* file encoding = "UTF-8" */
+				FileInputStream fileInputStream = new FileInputStream(new File(dir, files[i]));
+				InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8");
+				BufferedReader in = new BufferedReader(inputStreamReader);
+				while ((help = in.readLine()) != null) {
+					help = help.replaceAll("&", "&amp;");
+					help = help.replaceAll("<", "&lt;");
+					help = help.replaceAll(">", "&gt;");
+					help = help.replaceAll("\"", "&quot;");
+					text.addElement(help);
+				}
+				in.close();
+				inputStreamReader.close();
+				fileInputStream.close();
+			} catch (FileNotFoundException e) {
+				System.out.println("File not found: " + ((new File(dir, files[i])).toString()));
+			} catch (IOException e) {
+
+			}
+			result[i] = new String[text.size()];
+			text.copyInto(result[i]);
+		}
+		return result;
+	}
+
+	/*
+	 * Used by the "Report" class. All source files are returned as an array of
+	 * an array of chars.
+	 */
+	public char[][] readFilesChar(String[] files) {
+		char[][] result = new char[files.length][];
+
+		for (int i = 0; i < files.length; i++) {
+			try {
+				File file = new File(dir, files[i]);
+				int size = (int) file.length();
+				char[] buffer = new char[size];
+
+				FileReader fis = new FileReader(file);
+
+				if (size != fis.read(buffer)) {
+					System.out.println("Not right size read from the file, " + "but I will still continue...");
+				}
+
+				result[i] = buffer;
+				fis.close();
+			} catch (FileNotFoundException e) {
+				// TODO: Should an ExitException be thrown here?
+				System.out.println("File not found: " + ((new File(dir, files[i])).toString()));
+			} catch (IOException e) {
+
+			}
+		}
+		return result;
+	}
+
+	public int compareTo(Submission other) {
+		return name.compareTo(other.name);
+	}
+
+	public String toString() {
+		return name;
+	}
+}

jplag/src/test/resources/samples/PartialPlagiarism/C/Token.java

@@ -0,0 +1,47 @@
+import java.io.Serializable;
+
+abstract public class Token implements TokenConstants, Serializable {
+	private static final long serialVersionUID = 862935679966383302L;
+	public int type;
+	public String file;
+	
+	protected boolean marked;
+	protected boolean basecode = false;
+	protected int hash = -1;// hash-value. set and used by main algorithm (GSTiling)
+	
+	public Token(int type, String file, int line) {
+		this(type, file, line, -1, -1);
+	}
+	
+	public Token(int type, String file, int line, int column, int length) {
+		this.type = type;
+		this.file = file;
+		setLine(line>0 ? line : 1);
+		setColumn(column);
+		setLength(length);
+	}
+	
+	// abstract members
+	abstract public int getLine();
+	abstract public int getColumn();
+	abstract public int getLength();
+	protected void setLine(int line) {}
+	protected void setColumn(int line) {}
+	protected void setLength(int line) {}
+	
+	// this is made to distinguish the character front end.
+	// maybe other front ends can use it too?
+	protected int getIndex() { return -1; }
+	
+	public static String type2string(int type) {
+		return "<abstract>";
+	}
+	
+	public String toString() {
+		return type2string(type);
+	}
+	
+	public static int numberOfTokens() { 
+		return 1;
+	} 
+}

jplag/src/test/resources/samples/PartialPlagiarism/C/TokenConstants.java

@@ -0,0 +1,8 @@
+public interface TokenConstants {
+  final static int FILE_END = 0;
+	/**
+	 * Used to optionally separate methods from each other with an always marked
+	 * token
+	 */
+  final static int SEPARATOR_TOKEN = 1;
+}

jplag/src/test/resources/samples/PartialPlagiarism/D/Match.java

@@ -0,0 +1,54 @@
+package samples.PartialPlagiarism.D;
+
+public class Match {
+  public int startA;
+  public int startB;
+  public int length;
+
+  public Match(int startA, int startB, int length) {
+    this.startA = startA;
+    this.startB = startB;
+    this.length = length;
+  }
+  public Match() {}
+
+  public void set(int startA, int startB, int length) {
+    this.startA = startA;
+    this.startB = startB;
+    this.length = length;
+  }
+
+  public final boolean contains(int index, int sub) {
+    int start = (sub==0 ? startA : startB);
+    return (start <= index && index < (start+length));
+  }
+
+  public final boolean overlap(Match other) {
+    if (startA < other.startA) {
+      if ((other.startA - startA) < length) return true;
+    } else {
+      if ((startA - other.startA) < other.length) return true;
+    }
+    if (startB < other.startB) {
+      if ((other.startB - startB) < length) return true;
+    } else {
+      if ((startB - other.startB) < other.length) return true;
+    }
+    return false;
+  }
+
+  public final boolean overlap(int oStartA, int oStartB, int oLength) {
+    if (startA < oStartA) {
+      if ((oStartA - startA) < length) return true;
+    } else {
+      if ((startA - oStartA) < oLength) return true;
+    }
+    if (startB < oStartB) {
+      if ((oStartB - startB) < length) return true;
+    } else {
+      if ((startB - oStartB) < oLength) return true;
+    }
+    return false;
+  }
+}
+

jplag/src/test/resources/samples/PartialPlagiarism/D/Matches.java

@@ -0,0 +1,59 @@
+package samples.PartialPlagiarism.D;
+
+/* Minimal class that stores "Match" objects.
+ */
+public class Matches {
+  public Match[] matches;
+
+  private int anzahl;
+  private final int increment = 20;
+  
+  public Matches() {
+    matches = new Match[10];
+    for (int i=0; i<10; i++) matches[i] = new Match();
+    anzahl = 0;
+  }
+
+  public final int size() {
+    return anzahl;
+  }
+
+  public final void ensureCapacity(int minCapacity) {
+    int oldCapacity = matches.length;
+    if (minCapacity > oldCapacity) {
+      Match[] oldMatches = matches;
+      int newCapacity = (oldCapacity + increment);
+      if (newCapacity < minCapacity) {
+	newCapacity = minCapacity;
+      }
+      matches = new Match[newCapacity];
+      System.arraycopy(oldMatches, 0, matches, 0, oldCapacity);
+      for (int i=oldCapacity; i<newCapacity; i++)
+	matches[i] = new Match();
+    }
+  }
+  /*
+  public final void addMatch(Match match) {
+    for (int i=0; i<anzahl; i++)
+      if (match.overlap(matches[i])) return;  // do not allow overlaps
+    ensureCapacity(anzahl + 1);
+    matches[anzahl++] = match;
+  }
+  */
+  public final void addMatch(int startA, int startB, int length) {
+    for (int i=anzahl-1; i>=0; i--) { // starting at the end is better(?)
+      if (matches[i].overlap(startA,startB,length)) return;
+                                      // no overlaps!
+    }
+    ensureCapacity(anzahl + 1);
+    //if (matches[anzahl] != null)  // object recycling...
+    matches[anzahl].set(startA,startB,length);
+    //else
+    //matches[anzahl ] = new Match(startA,startB,length);
+    anzahl++;
+  }
+
+  public final void clear() {
+    anzahl=0;
+  }
+}

jplag/src/test/resources/samples/PartialPlagiarism/D/Structure.java

@@ -0,0 +1,120 @@
+package samples.PartialPlagiarism.D;
+
+import java.io.*;
+
+/** The tokenlist */
+public class Structure implements TokenConstants {
+	public Token[] tokens = new Token[0];
+	Table table = null;
+	int hash_length = -1;
+
+	int files; // number of END_FILE tokens
+	private int anzahl;
+
+	//private final int increment = 300;
+
+	public Structure() {
+		tokens = new Token[400];
+		files = anzahl = 0;
+	}
+
+	public final int size() {
+		return anzahl;
+	}
+
+	public final void ensureCapacity(int minCapacity) {
+		int oldCapacity = tokens.length;
+		if (minCapacity > oldCapacity) {
+			Token[] oldTokens = tokens;
+			int newCapacity = (2 * oldCapacity);
+			if (newCapacity < minCapacity) {
+				newCapacity = minCapacity;
+			}
+			tokens = new Token[newCapacity];
+			System.arraycopy(oldTokens, 0, tokens, 0, anzahl);
+		}
+	}
+
+	public final void addToken(Token token) {
+		ensureCapacity(anzahl + 1);
+		if (anzahl > 0 && tokens[anzahl - 1].file.equals(token.file))
+			token.file = tokens[anzahl - 1].file; // To save memory ...
+		if ((anzahl > 0) && (token.getLine() < tokens[anzahl - 1].getLine()) && (token.file.equals(tokens[anzahl - 1].file)))
+			token.setLine(tokens[anzahl - 1].getLine());
+		// just to make sure
+
+		tokens[anzahl++] = token;
+		if (token.type == FILE_END)
+			files++;
+	}
+
+	public final String toString() {
+		StringBuffer buf = new StringBuffer();
+
+		try {
+			for (int i = 0; i < anzahl; i++) {
+				String s = tokens[i].toString();
+				buf.append(i);
+				buf.append("\t");
+				buf.append(s);
+				if (i < anzahl - 1) {
+					buf.append("\n");
+				}
+			}
+		} catch (OutOfMemoryError e) {
+			return "Tokenlist to large for output: " + (anzahl) + " Tokens";
+		}
+		return buf.toString();
+	}
+
+	public void save(File file) {
+
+		try {
+			ObjectOutputStream p = new ObjectOutputStream(/* new GZIPOutputStream */(new FileOutputStream(file)));
+
+			p.writeInt(anzahl);
+			p.writeInt(hash_length);
+			p.writeInt(files);
+
+			for (int i = 0; i < anzahl; i++)
+				p.writeObject(tokens[i]);
+			p.flush();
+			p.close();
+		} catch (IOException e) {
+			System.out.println("Error writing file: " + file.toString());
+		}
+	}
+
+	/* returns "true" when successful */
+	public boolean load(File file) {
+		try {
+			ObjectInputStream p = new ObjectInputStream(/* new GZIPInputStream */(new FileInputStream(file)));
+
+			int newAnzahl = p.readInt();
+			hash_length = p.readInt();
+			files = p.readInt();
+			ensureCapacity(newAnzahl);
+			anzahl = newAnzahl;
+			for (int i = 0; i < anzahl; i++) {
+				tokens[i] = (Token) p.readObject();
+				// special case for text tokens:
+				// 	if (tokens[i] instanceof jplag.text.TextToken) {
+				// 	  jplag.text.TextToken token = (jplag.text.TextToken)tokens[i];
+				// 	  jplag.text.TextToken.put(token.getText(), token.type);
+				// 	}
+			}
+			p.close();
+			table = null;
+		} catch (FileNotFoundException e) {
+			System.out.println("File not found: " + file.toString());
+			return false;
+		} catch (IOException e) {
+			System.out.println("Error reading file: " + file.toString() + " (" + e + ")");
+			return false;
+		} catch (ClassNotFoundException e) {
+			System.out.println("Class not found in file: " + file.toString());
+			return false;
+		}
+		return true;
+	}
+}

jplag/src/test/resources/samples/PartialPlagiarism/D/Submission.java

@@ -0,0 +1,206 @@
+package samples.PartialPlagiarism.D;
+
+import java.io.*;
+import java.net.URL;
+import java.text.DecimalFormat;
+import java.util.Vector;
+
+/*
+ * Everything about a single submission is stored in this object. (directory,
+ * files, ...)
+ */
+public class Submission implements Comparable<Submission> {
+	public String name;
+
+	private boolean readSubDirs;
+
+	public File dir;
+
+	public String[] files = null; // = new String[0];
+
+	public Structure structure;
+
+	public int structSize = 0;
+
+	// public long structMem;
+	boolean exact_match = false; // used for fallback
+
+	public boolean errors = false;
+
+	public DecimalFormat format = new DecimalFormat("0000");
+
+	public Submission(String name, File dir, boolean readSubDirs) {
+		this.dir = dir;
+		this.name = name;
+		this.readSubDirs = readSubDirs;
+		try {
+			lookupDir(dir, "");
+		} catch (Throwable b) {
+		}
+	}
+
+	public Submission(String name, File dir) {
+		this.dir = dir;
+		this.name = name;
+		this.readSubDirs = false;
+
+		files = new String[1];
+		files[0] = name;
+
+	}
+
+	// recursively read in all the files
+	private void lookupDir(File dir, String subDir) throws Throwable {
+		File aktDir = new File(dir, subDir);
+		if (!aktDir.isDirectory())
+			return;
+		if (readSubDirs) {
+			String[] dirs = aktDir.list();
+			if (!subDir.equals(""))
+				for (int i = 0; i < dirs.length; i++)
+					lookupDir(dir, subDir + File.separator + dirs[i]);
+			else
+				for (int i = 0; i < dirs.length; i++)
+					lookupDir(dir, dirs[i]);
+		}
+		String[] newFiles = aktDir.list(new FilenameFilter() {
+			public boolean accept(File dir, String name) {
+					return false;
+			}
+		});
+		if (files != null) {
+			String[] oldFiles = files;
+			files = new String[oldFiles.length + newFiles.length];
+			if (subDir != "")
+				for (int i = 0; i < newFiles.length; i++)
+					files[i] = subDir + File.separator + newFiles[i];
+			else
+				System.arraycopy(newFiles, 0, files, 0, newFiles.length);
+			System.arraycopy(oldFiles, 0, files, newFiles.length, oldFiles.length);
+		} else {
+			if (subDir != "") {
+				files = new String[newFiles.length];
+				for (int i = 0; i < newFiles.length; i++)
+					files[i] = subDir + File.separator + newFiles[i];
+			} else
+				files = newFiles;
+		}
+	}
+
+	/* parse all the files... */
+	public boolean parse(){
+		return false;
+	}
+
+	/*
+	 * This method is used to copy files that can not be parsed to a special
+	 * folder: jplag/errors/java old_java scheme cpp /001/(...files...)
+	 * /002/(...files...)
+	 */
+	private void doSmth() {
+		File failureDir = null;
+		try {
+			URL url = Submission.class.getResource("errors");
+			failureDir = new File(url.getFile());
+		} catch (NullPointerException e) {
+			return;
+		}
+
+		if (!failureDir.exists())
+			failureDir.mkdir();
+		int i = 0;
+	}
+
+	private void copyFile(File in, File out) {
+		byte[] buffer = new byte[10000];
+		try {
+			FileInputStream des = new FileInputStream(in);
+			FileOutputStream foo = new FileOutputStream(out);
+			int count;
+			do {
+				count = des.read(buffer);
+				if (count != -1)
+					foo.write(buffer, 0, count);
+			} while (count != -1);
+			des.close();
+			foo.close();
+		} catch (IOException e) {
+			System.out.println("Oh no!");
+		}
+	}
+
+	public int size() {
+		if (structure != null)
+			return structSize = structure.size();
+		return structSize;
+	}
+
+	/*
+	 * Used by the "Report" class. All source files are returned as an array of
+	 * an array of strings.
+	 */
+	public String[][] readFiles(String[] files) {
+		String[][] newName = new String[files.length][];
+		String help;
+
+		Vector<String> text = new Vector<String>();
+		for (int i = 0; i < files.length; i++) {
+			text.removeAllElements();
+			try {
+				/* file encoding = "UTF-8" */
+				FileInputStream fileInputStream = new FileInputStream(new File(dir, files[i]));
+				InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8");
+				BufferedReader in = new BufferedReader(inputStreamReader);
+				while ((help = in.readLine()) != null) {
+					help = help.replaceAll("&", "&amp;");
+					text.addElement(help);
+				}
+				in.close();
+				inputStreamReader.close();
+				fileInputStream.close();
+			} catch (FileNotFoundException e) {
+				System.out.println("File not found: " + ((new File(dir, files[i])).toString()));
+			} catch (IOException e) {
+				System.out.println("Oh no!");
+			}
+			newName[i] = new String[text.size()];
+			text.copyInto(newName[i]);
+		}
+		return newName;
+	}
+
+	/*
+	 * Used by the "Report" class. All source files are returned as an array of
+	 * an array of chars.
+	 */
+	public char[][] readFilesChar(String[] files) {
+		char[][] variabl = new char[files.length][];
+
+		for (int i = 0; i < files.length; i++) {
+			try {
+				File file = new File(dir, files[i]);
+				int size = (int) file.length();
+				char[] buffer = new char[size];
+
+				FileReader fis = new FileReader(file);
+
+				if (size != fis.read(buffer)) {
+					System.out.println("Not right size read from the file, " + "but I will still continue...");
+				}
+
+				variabl[i] = buffer;
+				fis.close();
+			} catch (FileNotFoundException e) {
+				System.out.println("Oh no");
+			} catch (IOException e) {
+
+			}
+		}
+		return variabl;
+	}
+
+	@Override
+	public int compareTo(Submission o) {
+		return 0;
+	}
+}

jplag/src/test/resources/samples/PartialPlagiarism/D/Token.java

@@ -0,0 +1,44 @@
+package samples.PartialPlagiarism.D;
+
+import java.io.Serializable;
+
+abstract public class Token implements TokenConstants, Serializable {
+	private static final long serialVersionUID = 862335679966383302L;
+	public int type;
+	public String file;
+	
+	protected boolean marked;
+	protected boolean basecode = false;
+	protected int hash = -1;// hash-value. set and used by main algorithm (GSTiling)
+	
+	public Token(int type, String file, int line, int column, int length) {
+		this.type = type;
+		this.file = file;
+		setLine(line>0 ? line : 1);
+		setColumn(column);
+		setLength(length);
+	}
+	
+	// abstract members
+	abstract public int getLine();
+	abstract public int getColumn();
+	abstract public int getLength();
+	protected void setLine(int line) {}
+	protected void setColumn(int line) {}
+	protected void setLength(int line) {}
+	
+	// This is original work not copied
+	protected int getIndex() { return -1; }
+	
+	public static String type2string(int type) {
+		return "<abstract>";
+	}
+	
+	public String toString() {
+		return type2string(type);
+	}
+	
+	public static int numberOfTokens() { 
+		return 1;
+	} 
+}

jplag/src/test/resources/samples/PartialPlagiarism/D/TokenConstants.java

@@ -0,0 +1,12 @@
+package samples.PartialPlagiarism.D;
+
+public interface TokenConstants {
+  final static int FILE_END = 0;
+	/**
+	 * Used to optionally separate methods from each other with an always marked
+	 * token
+	 */
+  final static int SEPARATOR_TOKEN = 1;
+
+  final static int ANOTHER_TOKEN = 42;
+}

jplag/src/test/resources/samples/PartialPlagiarism/E/WhatAmIDoingHere.java

@@ -0,0 +1,5 @@
+public class WhatAmIDoingHere {
+    public static void main(String[] args) {
+        System.out.println("Hello JPlag!");
+    }
+}