diff options
6 files changed, 66 insertions, 29 deletions
diff --git a/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionFactory.java b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionFactory.java index 56fc052..17eddaf 100644 --- a/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionFactory.java +++ b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionFactory.java @@ -9,8 +9,8 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ConcurrentSkipListSet; import java.util.function.Consumer; import java.util.function.Supplier; @@ -25,20 +25,20 @@ public class ObjectCollectionFactory<T extends ObjectIdentifier> { private final FileIdentifierProvider<T> identifierFactory; private final int threads; private Exception exception = null; // fail in async + private final int minParallelProcessFileCountThreshold; - public ObjectCollectionFactory(FileIdentifierProvider<T> identifierFactory, int threads) { + public ObjectCollectionFactory(FileIdentifierProvider<T> identifierFactory, int threads, int minParallelProcessFileCountThreshold) { this.identifierFactory = identifierFactory; this.threads = threads; + this.minParallelProcessFileCountThreshold = minParallelProcessFileCountThreshold; if (threads <= 0) throw new IllegalArgumentException("thread count must be positive."); + if (minParallelProcessFileCountThreshold < 0) + throw new IllegalArgumentException("minParallelProcessFileCountThreshold must not be negative."); } public ObjectCollection fromDirectory(File directory, Set<String> ignoredFiles) throws IOException { - final int minParallelProcessFileCountThreshold = 0; - - final Set<ObjectElement> subFiles = new ConcurrentSkipListSet<>(); - final Map<String, ObjectCollection> subCollections = new HashMap<>(); if (!Objects.requireNonNull(directory).isDirectory()) @@ -61,6 +61,8 @@ public class ObjectCollectionFactory<T extends ObjectIdentifier> { } } + final Set<ObjectElement> subFiles = ConcurrentHashMap.newKeySet(files.size()); + // deal with all direct sub files if (threads == 1 || files.size() < minParallelProcessFileCountThreshold) { for (File file : files) { @@ -90,7 +92,7 @@ public class ObjectCollectionFactory<T extends ObjectIdentifier> { } } - return new ObjectCollection(directory.getName(), new HashSet<>(subFiles), subCollections); + return new ObjectCollection(directory.getName(), subFiles, subCollections); } public ObjectCollection fromDirectory(File directory) throws IOException { diff --git a/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectElement.java b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectElement.java index ee677fa..ff998f0 100644 --- a/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectElement.java +++ b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectElement.java @@ -1,7 +1,6 @@ package com.keuin.kbackupfabric.backup.incremental; import com.keuin.kbackupfabric.backup.incremental.identifier.ObjectIdentifier; -import org.jetbrains.annotations.NotNull; import java.io.Serializable; import java.util.Objects; @@ -10,7 +9,7 @@ import java.util.Objects; * Representing a file in a ObjectCollection. * Immutable. */ -public class ObjectElement implements Serializable, Comparable<ObjectElement> { +public class ObjectElement implements Serializable { private final String name; private final ObjectIdentifier identifier; @@ -59,8 +58,4 @@ public class ObjectElement implements Serializable, Comparable<ObjectElement> { '}'; } - @Override - public int compareTo(@NotNull ObjectElement objectElement) { - return name.compareTo(objectElement.name); - } } diff --git a/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java b/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java index ab0c114..569c4c6 100644 --- a/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java +++ b/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java @@ -46,7 +46,7 @@ public class ConfiguredIncrementalBackupMethod implements ConfiguredBackupMethod // construct incremental backup index PrintUtil.info("Hashing files..."); - ObjectCollection collection = new ObjectCollectionFactory<>(Sha256Identifier.getFactory(), hashFactoryThreads) + ObjectCollection collection = new ObjectCollectionFactory<>(Sha256Identifier.getFactory(), hashFactoryThreads, 16) .fromDirectory(levelPathFile, new HashSet<>(Arrays.asList("session.lock", "kbackup_metadata"))); // update storage diff --git a/src/test/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionFactoryTest.java b/src/test/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionFactoryTest.java index 16fb258..5052a82 100644 --- a/src/test/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionFactoryTest.java +++ b/src/test/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionFactoryTest.java @@ -26,28 +26,48 @@ public class ObjectCollectionFactoryTest { @Test public void fromDirectory1() { - fromDirectory(1); + fromDirectory(1, 0); } @Test public void fromDirectory2() { - fromDirectory(2); + fromDirectory(2, 0); } @Test public void fromDirectory4() { - fromDirectory(4); + fromDirectory(4, 0); } @Test public void fromDirectory8() { - fromDirectory(8); + fromDirectory(8, 0); } - public void fromDirectory(int threads) { + @Test + public void fromDirectory1A() { + fromDirectory(1, 1000); + } + + @Test + public void fromDirectory2A() { + fromDirectory(2, 1000); + } + + @Test + public void fromDirectory4A() { + fromDirectory(4, 1000); + } + + @Test + public void fromDirectory8A() { + fromDirectory(8, 1000); + } + + public void fromDirectory(int threads, int multiThreadThreshold) { try { ObjectCollectionFactory<Sha256Identifier> factory = - new ObjectCollectionFactory<>(Sha256Identifier.getFactory(), threads); + new ObjectCollectionFactory<>(Sha256Identifier.getFactory(), threads, multiThreadThreshold); ObjectCollection collection = factory.fromDirectory(new File("./testfile/ObjectCollectionFactoryTest")); diff --git a/src/test/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializerTest.java b/src/test/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializerTest.java index 33f5a9c..d21e5b3 100644 --- a/src/test/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializerTest.java +++ b/src/test/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializerTest.java @@ -13,27 +13,47 @@ public class ObjectCollectionSerializerTest { @Test public void testSerializationConsistency1() throws IOException { - testSerializationConsistency(1); + testSerializationConsistency(1, 0); } @Test public void testSerializationConsistency2() throws IOException { - testSerializationConsistency(2); + testSerializationConsistency(2, 0); } @Test public void testSerializationConsistency4() throws IOException { - testSerializationConsistency(4); + testSerializationConsistency(4, 0); } @Test public void testSerializationConsistency8() throws IOException { - testSerializationConsistency(8); + testSerializationConsistency(8, 0); } - public void testSerializationConsistency(int threads) throws IOException { + @Test + public void testSerializationConsistency1A() throws IOException { + testSerializationConsistency(1, 1000); + } + + @Test + public void testSerializationConsistency2A() throws IOException { + testSerializationConsistency(2, 1000); + } + + @Test + public void testSerializationConsistency4A() throws IOException { + testSerializationConsistency(4, 1000); + } + + @Test + public void testSerializationConsistency8A() throws IOException { + testSerializationConsistency(8, 1000); + } + + public void testSerializationConsistency(int threads, int multiThreadThreshold) throws IOException { ObjectCollectionFactory<Sha256Identifier> factory = - new ObjectCollectionFactory<>(Sha256Identifier.getFactory(), threads); + new ObjectCollectionFactory<>(Sha256Identifier.getFactory(), threads, multiThreadThreshold); ObjectCollection collection = factory.fromDirectory(new File("./testfile/ObjectCollectionFactoryTest")); File file = new File("./testfile/serialized"); diff --git a/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java b/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java index 2b4bd94..badc744 100644 --- a/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java +++ b/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java @@ -21,9 +21,9 @@ public class ConfiguredIncrementalBackupMethodTest { private final String destDirectoryName = "destination"; private final String indexFileName = "index"; - private final double directoryFactor = 0.03; - private final double fileFactor = 0.04; - private final int maxRandomFileSizeBytes = 1024 * 1024 * 4; + private final double directoryFactor = 0.05; + private final double fileFactor = 0.1; + private final int maxRandomFileSizeBytes = 1024 * 1024; private final Function<Integer, Integer> scaleDecayFunc = (x) -> x - 1; @Test |