From 5410762b8272feca0680f7a48f895714b91e3df7 Mon Sep 17 00:00:00 2001 From: Keuin Date: Wed, 13 Jan 2021 19:39:55 +0800 Subject: Add a more comprehensive test for incremental backup --- .../ConfiguredIncrementalBackupMethodTest.java | 193 +++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java (limited to 'src/test/java/com/keuin/kbackupfabric/operation/backup') diff --git a/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java b/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java new file mode 100644 index 0000000..7e4f267 --- /dev/null +++ b/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java @@ -0,0 +1,193 @@ +package com.keuin.kbackupfabric.operation.backup.method; + +import org.apache.commons.codec.digest.DigestUtils; +import org.junit.Test; + +import java.io.*; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.function.Function; + +import static org.apache.commons.io.FileUtils.forceDelete; +import static org.junit.Assert.*; + +public class ConfiguredIncrementalBackupMethodTest { + + private final String testTempPath = "R:\\"; + private final String sourceDirectoryName = "source"; + private final String destDirectoryName = "destination"; + private final String indexFileName = "index"; + + private final double directoryFactor = 0.4; + private final double fileFactor = 0.1; + private final int maxRandomFileSizeBytes = 1024 * 1024; + private final Function scaleDecayFunc = (x) -> x - 1; + + @Test + public void iterationTest() throws IOException { + int a = 100; + for (int i = 0; i < a; ++i) { + performTest(Math.min(i + 1, 10)); + System.out.println("Round " + i + " passed."); + } + } + + private void performTest(int scale) throws IOException { + + // init source and destination + final Path sourcePath = Paths.get(testTempPath, sourceDirectoryName); + final Path destPath = Paths.get(testTempPath, destDirectoryName); + if (new File(sourcePath.toString()).exists()) { + forceDelete(new File(sourcePath.toString())); + if (!new File(sourcePath.toString()).mkdirs()) + fail(); + } + if (new File(destPath.toString()).exists()) { + forceDelete(new File(destPath.toString())); + if (!new File(destPath.toString()).mkdirs()) + fail(); + } + if (new File(testTempPath, indexFileName).exists()) { + if (!new File(testTempPath, indexFileName).delete()) + fail(); + } + + // initialize src + createRandomDirectoryTree(sourcePath.toString(), scale); + + String hash1 = calcMD5HashForDir(sourcePath.toFile(), true); + + // copy src to dest + ConfiguredIncrementalBackupMethod method = new ConfiguredIncrementalBackupMethod( + indexFileName, + sourcePath.toString(), + testTempPath, + destPath.toString() + ); + method.backup(); + + // delete src + forceDelete(sourcePath.toFile()); + assertFalse(sourcePath.toFile().isDirectory()); + + // restore src + if (!method.restore()) + fail(); + + String hash2 = calcMD5HashForDir(sourcePath.toFile(), true); + + assertEquals(hash1, hash2); + } + + private void createRandomDirectoryTree(String path, int scale) throws IOException { + if (scale <= 0) { + if (Math.random() < 0.5) + if (!new File(path).mkdirs() && !new File(path).exists()) + throw new IOException("Failed to create directory " + path); + return; + } + if (!new File(path).isDirectory() && !new File(path).mkdirs()) + throw new IOException("Failed to create directory " + path); + + int subFileCount = (int) Math.round(Math.random() * 10 * scale * fileFactor); + for (int i = 0; i < subFileCount; i++) { + String subFile = null; + while (subFile == null || new File(path, subFile).exists()) + subFile = getRandomString((int) (Math.random() * 16 + 1)); + createRandomFile(new File(path, subFile), maxRandomFileSizeBytes); + } + + + int subDirCount = (int) Math.round(Math.random() * 10 * scale * directoryFactor); + for (int i = 0; i < subDirCount; i++) { + String subDir = null; + while (subDir == null || new File(path, subDir).exists()) + subDir = getRandomString((int) (Math.random() * 16 + 1)); + createRandomDirectoryTree(new File(path, subDir).getAbsolutePath(), scaleDecayFunc.apply(scale)); + } + } + + private static void createRandomFile(File file, int maxSizeBytes) throws IOException { + if (!file.createNewFile()) + throw new IOException("Failed to create file " + file.getAbsolutePath()); + try (FileOutputStream fileOutputStream = new FileOutputStream(file)) { + int fileBytes = (int) (maxSizeBytes * Math.random() + 1); + Random random = new Random(); + final int chunkSize = 1024 * 4; + byte[] randomChunk = new byte[chunkSize]; + for (int i = 0; i < fileBytes / chunkSize; i++) { + random.nextBytes(randomChunk); + fileOutputStream.write(randomChunk); + } + if (fileBytes % chunkSize != 0) { + randomChunk = new byte[fileBytes % chunkSize]; + random.nextBytes(randomChunk); + fileOutputStream.write(randomChunk); + } + } + } + + private static String getRandomString(int length) { + String str = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + Random random = new Random(); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + int number = random.nextInt(62); + sb.append(str.charAt(number)); + } + return sb.toString(); + } + + public String calcMD5HashForDir(File dirToHash, boolean includeHiddenFiles) { + + assert (dirToHash.isDirectory()); + Vector fileStreams = new Vector<>(); + + System.out.println("Found files for hashing:"); + collectInputStreams(dirToHash, fileStreams, includeHiddenFiles); + + SequenceInputStream seqStream = + new SequenceInputStream(fileStreams.elements()); + + try { + String md5Hash = DigestUtils.md5Hex(seqStream); + seqStream.close(); + return md5Hash; + } catch (IOException e) { + throw new RuntimeException("Error reading files to hash in " + + dirToHash.getAbsolutePath(), e); + } + + } + + private void collectInputStreams(File dir, + List foundStreams, + boolean includeHiddenFiles) { + + File[] fileList = dir.listFiles(); + Arrays.sort(fileList, // Need in reproducible order + new Comparator() { + public int compare(File f1, File f2) { + return f1.getName().compareTo(f2.getName()); + } + }); + + for (File f : fileList) { + if (!includeHiddenFiles && f.getName().startsWith(".")) { + // Skip it + } else if (f.isDirectory()) { + collectInputStreams(f, foundStreams, includeHiddenFiles); + } else { + try { + System.out.println("\t" + f.getAbsolutePath()); + foundStreams.add(new FileInputStream(f)); + } catch (FileNotFoundException e) { + throw new AssertionError(e.getMessage() + + ": file should never not be found!"); + } + } + } + + } +} \ No newline at end of file -- cgit v1.2.3 From 6baece29cccc906651331cbf8a90a06f8cee045b Mon Sep 17 00:00:00 2001 From: Keuin Date: Thu, 14 Jan 2021 13:00:52 +0800 Subject: Fix a minor naming bug. Code refactor. Improve test. --- .../java/com/keuin/kbackupfabric/KBCommands.java | 4 +- .../com/keuin/kbackupfabric/util/BytesUtil.java | 17 ++++++- .../util/backup/BackupFilesystemUtil.java | 4 -- .../identifier/FileIdentifierProvider.java | 7 +++ .../incremental/identifier/Sha256Identifier.java | 20 ++++++++- .../identifier/SingleHashIdentifier.java | 16 ++++++- .../identifier/StorageObjectLoader.java | 25 +++++++++++ .../manager/IncrementalBackupStorageManager.java | 52 +++++++++++++++++++++- .../util/backup/name/BackupFileNameEncoder.java | 8 +++- .../name/IncrementalBackupFileNameEncoder.java | 2 +- .../name/PrimitiveBackupFileNameEncoder.java | 2 +- .../backup/provider/AvailableBackupProvider.java | 8 ++++ .../provider/IncrementalBackupInformation.java | 13 ++++++ .../provider/PrimitiveBackupInformation.java | 14 ++++++ .../ConfiguredIncrementalBackupMethodTest.java | 34 +++++++++++--- .../incremental/ObjectCollectionFactoryTest.java | 14 +++--- .../identifier/Sha256IdentifierTest.java | 2 +- .../name/IncrementalBackupFileNameEncoderTest.java | 38 ++++++++++++++++ .../name/PrimitiveBackupFileNameEncoderTest.java | 33 +++++++++++++- 19 files changed, 281 insertions(+), 32 deletions(-) create mode 100644 src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/StorageObjectLoader.java create mode 100644 src/main/java/com/keuin/kbackupfabric/util/backup/provider/AvailableBackupProvider.java create mode 100644 src/main/java/com/keuin/kbackupfabric/util/backup/provider/IncrementalBackupInformation.java create mode 100644 src/main/java/com/keuin/kbackupfabric/util/backup/provider/PrimitiveBackupInformation.java create mode 100644 src/test/java/com/keuin/kbackupfabric/util/backup/name/IncrementalBackupFileNameEncoderTest.java (limited to 'src/test/java/com/keuin/kbackupfabric/operation/backup') diff --git a/src/main/java/com/keuin/kbackupfabric/KBCommands.java b/src/main/java/com/keuin/kbackupfabric/KBCommands.java index 847fe7c..98f6369 100644 --- a/src/main/java/com/keuin/kbackupfabric/KBCommands.java +++ b/src/main/java/com/keuin/kbackupfabric/KBCommands.java @@ -84,7 +84,9 @@ public final class KBCommands { public static int list(CommandContext context) { MinecraftServer server = context.getSource().getMinecraftServer(); File[] files = getBackupSaveDirectory(server).listFiles( - (dir, name) -> dir.isDirectory() && name.toLowerCase().endsWith(".zip") && name.toLowerCase().startsWith(getBackupFileNamePrefix()) + (dir, name) -> dir.isDirectory() && + (name.toLowerCase().endsWith(".zip") && name.toLowerCase().startsWith(getBackupFileNamePrefix()) + || name.toLowerCase().endsWith(".kbi")) ); synchronized (backupFileNameList) { diff --git a/src/main/java/com/keuin/kbackupfabric/util/BytesUtil.java b/src/main/java/com/keuin/kbackupfabric/util/BytesUtil.java index 6ded7b8..c33c028 100644 --- a/src/main/java/com/keuin/kbackupfabric/util/BytesUtil.java +++ b/src/main/java/com/keuin/kbackupfabric/util/BytesUtil.java @@ -4,6 +4,7 @@ import java.nio.charset.StandardCharsets; public class BytesUtil { private static final byte[] HEX_ARRAY = "0123456789ABCDEF".getBytes(StandardCharsets.US_ASCII); + public static String bytesToHex(byte[] bytes) { byte[] hexChars = new byte[bytes.length * 2]; for (int j = 0; j < bytes.length; j++) { @@ -11,6 +12,20 @@ public class BytesUtil { hexChars[j * 2] = HEX_ARRAY[v >>> 4]; hexChars[j * 2 + 1] = HEX_ARRAY[v & 0x0F]; } - return new String(hexChars, StandardCharsets.UTF_8); + return new String(hexChars, StandardCharsets.UTF_8).toUpperCase(); + } + + public static byte[] hexToBytes(String s) { + int len = s.length(); + if (len % 2 != 0) + throw new IllegalArgumentException("Invalid hex string."); + byte[] b = new byte[len / 2]; + int index, v; + for (int i = 0; i < b.length; i++) { + index = i * 2; + v = Integer.parseInt(s.substring(index, index + 2), 16); + b[i] = (byte) v; + } + return b; } } diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/BackupFilesystemUtil.java b/src/main/java/com/keuin/kbackupfabric/util/backup/BackupFilesystemUtil.java index 837b638..7106ad2 100644 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/BackupFilesystemUtil.java +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/BackupFilesystemUtil.java @@ -23,10 +23,6 @@ public final class BackupFilesystemUtil { return backupFileNamePrefix; } - @Deprecated - public static String getBackupFileName(String backupName) { - return backupFileNamePrefix + backupName + ".zip"; - } // @Deprecated // public static String getBackupName(String backupFileName) { diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/FileIdentifierProvider.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/FileIdentifierProvider.java index 9a03371..3fbe284 100644 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/FileIdentifierProvider.java +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/FileIdentifierProvider.java @@ -4,5 +4,12 @@ import java.io.File; import java.io.IOException; public interface FileIdentifierProvider { + /** + * Generate file identifier from a random file. The file is not necessarily in the object base. + * + * @param file the file. + * @return the file identifier. + * @throws IOException when an I/O error occurs. + */ T fromFile(File file) throws IOException; } diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/Sha256Identifier.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/Sha256Identifier.java index cbf1bb9..c1c87e1 100644 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/Sha256Identifier.java +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/Sha256Identifier.java @@ -1,5 +1,7 @@ package com.keuin.kbackupfabric.util.backup.incremental.identifier; +import com.keuin.kbackupfabric.util.BytesUtil; + import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -16,20 +18,34 @@ public class Sha256Identifier extends SingleHashIdentifier { private static final int SHA256_LENGTH = 32; private static final Sha256Identifier DUMMY = new Sha256Identifier(new byte[SHA256_LENGTH]); // only for using its hash method private static final FileIdentifierProvider factory = Sha256Identifier::fromFile; + private static final String marker = "S2"; public static Sha256Identifier fromFile(File file) throws IOException { - if (!Objects.requireNonNull(file).isFile()) { + if (!file.isFile()) { throw new IllegalArgumentException("file is not a file"); } return new Sha256Identifier(DUMMY.hash(file)); } + /** + * Load sha-256 from a named file. Only used in StorageObjectLoader. + * + * @param fileName the file name. + * @return identifier. + */ + static Sha256Identifier fromFileName(String fileName) { + if (!fileName.matches(marker + "-[0-9A-Fa-f]{32}")) + return null; + String hexString = fileName.substring(marker.length() + 1); + return new Sha256Identifier(BytesUtil.hexToBytes(hexString)); + } + public static FileIdentifierProvider getFactory() { return factory; } protected Sha256Identifier(byte[] hash) { - super(hash); + super(hash, marker); Objects.requireNonNull(hash); if (hash.length != SHA256_LENGTH) { throw new IllegalStateException(String.format("SHA256 must be %d bytes", SHA256_LENGTH)); diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/SingleHashIdentifier.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/SingleHashIdentifier.java index 3b96f79..0f62f2b 100644 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/SingleHashIdentifier.java +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/SingleHashIdentifier.java @@ -5,6 +5,7 @@ import com.keuin.kbackupfabric.util.BytesUtil; import java.io.File; import java.io.IOException; import java.util.Arrays; +import java.util.Objects; /** * A simple identifier based on a single hash function. @@ -13,9 +14,13 @@ import java.util.Arrays; public abstract class SingleHashIdentifier implements ObjectIdentifier { private final byte[] hash; + private final String type; - protected SingleHashIdentifier(byte[] hash) { + protected SingleHashIdentifier(byte[] hash, String type) { + Objects.requireNonNull(hash); + Objects.requireNonNull(type); this.hash = Arrays.copyOf(hash, hash.length); + this.type = type; } /** @@ -28,7 +33,7 @@ public abstract class SingleHashIdentifier implements ObjectIdentifier { @Override public String getIdentification() { - return BytesUtil.bytesToHex(hash); + return type + "-" + BytesUtil.bytesToHex(hash); } @Override @@ -38,4 +43,11 @@ public abstract class SingleHashIdentifier implements ObjectIdentifier { } return Arrays.equals(hash, ((SingleHashIdentifier) obj).hash); } + + @Override + public int hashCode() { + int result = Objects.hash(type); + result = 31 * result + Arrays.hashCode(hash); + return result; + } } diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/StorageObjectLoader.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/StorageObjectLoader.java new file mode 100644 index 0000000..96bc295 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/StorageObjectLoader.java @@ -0,0 +1,25 @@ +package com.keuin.kbackupfabric.util.backup.incremental.identifier; + +import java.io.File; +import java.util.Objects; + +public class StorageObjectLoader { + /** + * Get identifier from storage file. + * + * @param file storage file. + * @return identifier. If failed, return null. + */ + public static ObjectIdentifier asIdentifier(File file) { + Objects.requireNonNull(file); + String fileName = file.getName(); + ObjectIdentifier identifier; + + identifier = Sha256Identifier.fromFileName(fileName); + if (identifier != null) + return identifier; + + // Add more identifiers. + return null; + } +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/manager/IncrementalBackupStorageManager.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/manager/IncrementalBackupStorageManager.java index 6c359c6..6fd339b 100644 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/manager/IncrementalBackupStorageManager.java +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/manager/IncrementalBackupStorageManager.java @@ -3,6 +3,8 @@ package com.keuin.kbackupfabric.util.backup.incremental.manager; import com.keuin.kbackupfabric.util.PrintUtil; import com.keuin.kbackupfabric.util.backup.incremental.ObjectCollection; import com.keuin.kbackupfabric.util.backup.incremental.ObjectElement; +import com.keuin.kbackupfabric.util.backup.incremental.identifier.ObjectIdentifier; +import com.keuin.kbackupfabric.util.backup.incremental.identifier.StorageObjectLoader; import java.io.File; import java.io.FileNotFoundException; @@ -10,14 +12,15 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Map; -import java.util.Objects; +import java.util.*; import static org.apache.commons.io.FileUtils.forceDelete; public class IncrementalBackupStorageManager { private final Path backupStorageBase; + private final Map map = new HashMap<>(); + private boolean loaded = false; public IncrementalBackupStorageManager(Path backupStorageBase) { this.backupStorageBase = backupStorageBase; @@ -127,8 +130,31 @@ public class IncrementalBackupStorageManager { return copyCount; } + public int cleanUnusedObjects(Iterable collectionIterable) { + // construct object list in memory + Set objects = new HashSet<>(); +// backupStorageBase + + for (ObjectCollection collection : collectionIterable) { + for (ObjectElement ele : collection.getElementMap().values()) { + + } + } + throw new RuntimeException("not impl"); + } + + /** + * Check all objects, return unused ones. + * + * @return the unused ones. + */ + private Map markUnusedObjects() { + throw new RuntimeException("not impl"); + } + /** * Check if the backup base contains given element. + * * @param objectElement the element. * @return true or false. */ @@ -137,4 +163,26 @@ public class IncrementalBackupStorageManager { return (new File(backupStorageBase.toFile(), objectElement.getIdentifier().getIdentification())).exists(); } + private void lazyLoadStorage() throws IOException { + if (!loaded) { + loadStorage(); + loaded = true; + } + } + + private synchronized void loadStorage() throws IOException { + map.clear(); + Files.walk(backupStorageBase, 1).forEach(path -> { + File file = path.toFile(); + ObjectIdentifier identifier = StorageObjectLoader.asIdentifier(file); + if (identifier == null) { + map.clear(); + throw new IllegalStateException(String.format( + "Bad storage object %s: cannot recognize identifier.", file.getName() + )); + } + map.put(identifier, file); + }); + } + } diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/name/BackupFileNameEncoder.java b/src/main/java/com/keuin/kbackupfabric/util/backup/name/BackupFileNameEncoder.java index 972403b..8ebc7ff 100644 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/name/BackupFileNameEncoder.java +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/name/BackupFileNameEncoder.java @@ -18,13 +18,19 @@ public interface BackupFileNameEncoder { /** * Extract custom and backup time from backup file name. + * * @param fileName the backup file name. * @return the information. If the given file name is invalid, return null. */ BackupBasicInformation decode(String fileName); + default boolean isValidFileName(String fileName) { + return decode(fileName) != null; + } + /** * Check if the given string is a valid custom backup name. + * * @param customName the custom backup name. * @return if the name is valid. */ @@ -45,7 +51,7 @@ public interface BackupFileNameEncoder { private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm.ss"); - BackupBasicInformation(String customName, LocalDateTime time) { + protected BackupBasicInformation(String customName, LocalDateTime time) { this.customName = customName; this.time = time; } diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/name/IncrementalBackupFileNameEncoder.java b/src/main/java/com/keuin/kbackupfabric/util/backup/name/IncrementalBackupFileNameEncoder.java index 3c35201..926f47c 100644 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/name/IncrementalBackupFileNameEncoder.java +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/name/IncrementalBackupFileNameEncoder.java @@ -24,7 +24,7 @@ public class IncrementalBackupFileNameEncoder implements BackupFileNameEncoder { @Override public BackupFileNameEncoder.BackupBasicInformation decode(String fileName) { Pattern pattern = Pattern.compile( - backupFileNamePrefix + "-" + "([0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2})_(.+)\\.kbi" + "^" + backupFileNamePrefix + "-" + "([0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2})_(.+)\\.kbi" + "$" ); Matcher matcher = pattern.matcher(fileName); if (matcher.find()) { diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/name/PrimitiveBackupFileNameEncoder.java b/src/main/java/com/keuin/kbackupfabric/util/backup/name/PrimitiveBackupFileNameEncoder.java index bcba114..ef15ae7 100644 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/name/PrimitiveBackupFileNameEncoder.java +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/name/PrimitiveBackupFileNameEncoder.java @@ -20,7 +20,7 @@ public class PrimitiveBackupFileNameEncoder implements BackupFileNameEncoder { @Override public BackupBasicInformation decode(String fileName) { Pattern pattern = Pattern.compile( - backupFileNamePrefix + "-" + "([0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2})_(.+)\\.zip" + "^" + backupFileNamePrefix + "-" + "([0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2})_(.+)\\.zip" + "$" ); Matcher matcher = pattern.matcher(fileName); if (matcher.find()) { diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/provider/AvailableBackupProvider.java b/src/main/java/com/keuin/kbackupfabric/util/backup/provider/AvailableBackupProvider.java new file mode 100644 index 0000000..caa0e84 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/provider/AvailableBackupProvider.java @@ -0,0 +1,8 @@ +package com.keuin.kbackupfabric.util.backup.provider; + +/** + * List all backup in disk. Provide their basic information as soon as possible. + */ +public class AvailableBackupProvider { + // TODO: remove obsolete impl. in command user interface. Use this instead. +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/provider/IncrementalBackupInformation.java b/src/main/java/com/keuin/kbackupfabric/util/backup/provider/IncrementalBackupInformation.java new file mode 100644 index 0000000..861d210 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/provider/IncrementalBackupInformation.java @@ -0,0 +1,13 @@ +package com.keuin.kbackupfabric.util.backup.provider; + +import com.keuin.kbackupfabric.util.backup.name.BackupFileNameEncoder; + +import java.time.LocalDateTime; + +public class IncrementalBackupInformation extends BackupFileNameEncoder.BackupBasicInformation { + // TODO: show total size for incremental backup + + public IncrementalBackupInformation(String customName, LocalDateTime time) { + super(customName, time); + } +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/provider/PrimitiveBackupInformation.java b/src/main/java/com/keuin/kbackupfabric/util/backup/provider/PrimitiveBackupInformation.java new file mode 100644 index 0000000..d3d2db8 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/provider/PrimitiveBackupInformation.java @@ -0,0 +1,14 @@ +package com.keuin.kbackupfabric.util.backup.provider; + +import com.keuin.kbackupfabric.util.backup.name.BackupFileNameEncoder; + +import java.time.LocalDateTime; + +public class PrimitiveBackupInformation extends BackupFileNameEncoder.BackupBasicInformation { + public final long sizeBytes; + + public PrimitiveBackupInformation(String customName, LocalDateTime time, long sizeBytes) { + super(customName, time); + this.sizeBytes = sizeBytes; + } +} diff --git a/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java b/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java index 7e4f267..58e1711 100644 --- a/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java +++ b/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java @@ -4,6 +4,7 @@ import org.apache.commons.codec.digest.DigestUtils; import org.junit.Test; import java.io.*; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -19,14 +20,14 @@ public class ConfiguredIncrementalBackupMethodTest { private final String destDirectoryName = "destination"; private final String indexFileName = "index"; - private final double directoryFactor = 0.4; - private final double fileFactor = 0.1; - private final int maxRandomFileSizeBytes = 1024 * 1024; + private final double directoryFactor = 0.03; + private final double fileFactor = 0.05; + private final int maxRandomFileSizeBytes = 1024 * 1024 * 16; private final Function scaleDecayFunc = (x) -> x - 1; @Test public void iterationTest() throws IOException { - int a = 100; + int a = 12; for (int i = 0; i < a; ++i) { performTest(Math.min(i + 1, 10)); System.out.println("Round " + i + " passed."); @@ -75,9 +76,28 @@ public class ConfiguredIncrementalBackupMethodTest { if (!method.restore()) fail(); + boolean fake = scale % 2 != 0; + + int[] success = new int[1]; + if (fake) { + Files.walk(sourcePath).filter(path -> path.toFile().isFile()).limit(3).forEach(path -> { + if (!path.toFile().delete()) + fail(); + success[0]++; + }); + if (success[0] == 0) + fake = false; + } + + if (fake) + System.out.println("Fake: deleted " + success[0] + " file(s)."); + String hash2 = calcMD5HashForDir(sourcePath.toFile(), true); - assertEquals(hash1, hash2); + if (!fake) + assertEquals(hash1, hash2); + else + assertNotEquals(hash1, hash2); } private void createRandomDirectoryTree(String path, int scale) throws IOException { @@ -94,7 +114,7 @@ public class ConfiguredIncrementalBackupMethodTest { for (int i = 0; i < subFileCount; i++) { String subFile = null; while (subFile == null || new File(path, subFile).exists()) - subFile = getRandomString((int) (Math.random() * 16 + 1)); + subFile = getRandomString((int) (Math.random() * 16 + 5)); createRandomFile(new File(path, subFile), maxRandomFileSizeBytes); } @@ -103,7 +123,7 @@ public class ConfiguredIncrementalBackupMethodTest { for (int i = 0; i < subDirCount; i++) { String subDir = null; while (subDir == null || new File(path, subDir).exists()) - subDir = getRandomString((int) (Math.random() * 16 + 1)); + subDir = getRandomString((int) (Math.random() * 16 + 5)); createRandomDirectoryTree(new File(path, subDir).getAbsolutePath(), scaleDecayFunc.apply(scale)); } } diff --git a/src/test/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectCollectionFactoryTest.java b/src/test/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectCollectionFactoryTest.java index b7e0aad..3f722a4 100644 --- a/src/test/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectCollectionFactoryTest.java +++ b/src/test/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectCollectionFactoryTest.java @@ -20,7 +20,7 @@ public class ObjectCollectionFactoryTest { } for (Map.Entry entry : collection.getElementMap().entrySet()) { // assertTrue(subElements.contains(e.getIdentification())); - assertEquals(subElements.get(entry.getKey()), entry.getValue().getIdentifier().getIdentification()); + assertEquals(subElements.get(entry.getKey()).toUpperCase(), entry.getValue().getIdentifier().getIdentification().toUpperCase()); } } @@ -39,14 +39,14 @@ public class ObjectCollectionFactoryTest { final Map elements = new HashMap<>(); // check root dir - elements.put("a", "261CA0D59FEE8FD169802BB8030A07CF23E5C1593FA81A16C6D0A8CF27DAA2ED"); - elements.put("b", "B3FED75012C4969DC63A50EBC4E745FF77E4A06E0B04720EF71EF033032EBAF7"); + elements.put("a", "S2-261CA0D59FEE8FD169802BB8030A07CF23E5C1593FA81A16C6D0A8CF27DAA2ED"); + elements.put("b", "S2-B3FED75012C4969DC63A50EBC4E745FF77E4A06E0B04720EF71EF033032EBAF7"); validate(collection, Arrays.asList("1", "2", "3"), elements); elements.clear(); // check `1` - elements.put("a", "E8620F35A5DB33B1257CC51245DDACDA8AF3E0D431A8A38473575E468BCBD0BD"); - elements.put("b", "19EE41585A674274891DE5A4B365DBAB9C49C576AB6F86CD515B683724D2DBBD"); + elements.put("a", "S2-E8620F35A5DB33B1257CC51245DDACDA8AF3E0D431A8A38473575E468BCBD0BD"); + elements.put("b", "S2-19EE41585A674274891DE5A4B365DBAB9C49C576AB6F86CD515B683724D2DBBD"); validate(collection.getSubCollectionMap().get("1"), Arrays.asList("11", "12"), elements); elements.clear(); @@ -60,8 +60,8 @@ public class ObjectCollectionFactoryTest { validate(collection.getSubCollectionMap().get("1").getSubCollectionMap().get("11"), Collections.singletonList("111"), Collections.emptyMap()); // check `111` - elements.put("a", "1EDBE882A757E1FAFCA77A9D3BE3FF5D2BB3E2037B238C865F1F957C431F43B4"); - elements.put("b", "30BA7CD8B4AD93A8B3826CD8D1518790924EEBB930EC04DF7DFB03A50B17D7BC"); + elements.put("a", "S2-1EDBE882A757E1FAFCA77A9D3BE3FF5D2BB3E2037B238C865F1F957C431F43B4"); + elements.put("b", "S2-30BA7CD8B4AD93A8B3826CD8D1518790924EEBB930EC04DF7DFB03A50B17D7BC"); validate( collection.getSubCollectionMap().get("1").getSubCollectionMap().get("11").getSubCollectionMap().get("111"), Collections.emptyList(), diff --git a/src/test/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/Sha256IdentifierTest.java b/src/test/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/Sha256IdentifierTest.java index 53e92e2..f799a95 100644 --- a/src/test/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/Sha256IdentifierTest.java +++ b/src/test/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/Sha256IdentifierTest.java @@ -15,7 +15,7 @@ public class Sha256IdentifierTest { try { Sha256Identifier sha256 = Sha256Identifier.fromFile(new File("./testfile/Sha256IdentifierTest")); String str = sha256.getIdentification().toUpperCase(); - assertEquals("315F5BDB76D078C43B8AC0064E4A0164612B1FCE77C869345BFC94C75894EDD3", str); + assertEquals("S2-315F5BDB76D078C43B8AC0064E4A0164612B1FCE77C869345BFC94C75894EDD3", str); } catch (IOException e) { e.printStackTrace(); fail(); diff --git a/src/test/java/com/keuin/kbackupfabric/util/backup/name/IncrementalBackupFileNameEncoderTest.java b/src/test/java/com/keuin/kbackupfabric/util/backup/name/IncrementalBackupFileNameEncoderTest.java new file mode 100644 index 0000000..e5fedd7 --- /dev/null +++ b/src/test/java/com/keuin/kbackupfabric/util/backup/name/IncrementalBackupFileNameEncoderTest.java @@ -0,0 +1,38 @@ +package com.keuin.kbackupfabric.util.backup.name; + +import org.junit.Test; + +import java.time.LocalDateTime; + +import static org.junit.Assert.*; + +public class IncrementalBackupFileNameEncoderTest { + @Test + public void testEncode() { + LocalDateTime time = LocalDateTime.of(1, 1, 1, 1, 1, 1); + String customName = "name"; + IncrementalBackupFileNameEncoder encoder = new IncrementalBackupFileNameEncoder(); + assertEquals("incremental-0001-01-01_01-01-01_name.kbi", encoder.encode(customName, time)); + } + + @Test + public void testDecode() { + LocalDateTime time = LocalDateTime.of(1, 1, 1, 1, 1, 1); + String customName = "name"; + IncrementalBackupFileNameEncoder encoder = new IncrementalBackupFileNameEncoder(); + BackupFileNameEncoder.BackupBasicInformation information = encoder.decode("incremental-0001-01-01_01-01-01_name.kbi"); + assertEquals(time, information.time); + assertEquals(customName, information.customName); + } + + @Test + public void isValid() { + IncrementalBackupFileNameEncoder encoder = new IncrementalBackupFileNameEncoder(); + assertTrue(encoder.isValidFileName("incremental-0001-01-01_01-01-01_name.kbi")); + assertTrue(encoder.isValidFileName("incremental-0001-01-01_01-01-01_0001-01-01_01-01-01_name.kbi")); + assertFalse(encoder.isValidFileName("incremental-0001-01-01_01-01-01incremental-0001-01-01_01-01-01_name.kbi")); + assertFalse(encoder.isValidFileName("incremental-0001-01-01_01-01-01_name")); + assertFalse(encoder.isValidFileName("incremental-0001-01-01_01-01-01_name.zip")); + assertFalse(encoder.isValidFileName("somefile")); + } +} \ No newline at end of file diff --git a/src/test/java/com/keuin/kbackupfabric/util/backup/name/PrimitiveBackupFileNameEncoderTest.java b/src/test/java/com/keuin/kbackupfabric/util/backup/name/PrimitiveBackupFileNameEncoderTest.java index 4823575..abc19d5 100644 --- a/src/test/java/com/keuin/kbackupfabric/util/backup/name/PrimitiveBackupFileNameEncoderTest.java +++ b/src/test/java/com/keuin/kbackupfabric/util/backup/name/PrimitiveBackupFileNameEncoderTest.java @@ -5,17 +5,46 @@ import org.junit.Test; import java.time.LocalDateTime; import java.time.ZoneOffset; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; public class PrimitiveBackupFileNameEncoderTest { @Test public void testConsistency() { - LocalDateTime time = LocalDateTime.ofEpochSecond(System.currentTimeMillis()/1000, 0, ZoneOffset.UTC); + LocalDateTime time = LocalDateTime.ofEpochSecond(System.currentTimeMillis() / 1000, 0, ZoneOffset.UTC); String name = "Test Na_me"; PrimitiveBackupFileNameEncoder encoder = new PrimitiveBackupFileNameEncoder(); BackupFileNameEncoder.BackupBasicInformation information = encoder.decode(encoder.encode(name, time)); assertEquals(time, information.time); assertEquals(name, information.customName); } + + @Test + public void testEncode() { + LocalDateTime time = LocalDateTime.of(1, 1, 1, 1, 1, 1); + String customName = "name"; + PrimitiveBackupFileNameEncoder encoder = new PrimitiveBackupFileNameEncoder(); + assertEquals("kbackup-0001-01-01_01-01-01_name.zip", encoder.encode(customName, time)); + } + + @Test + public void testDecode() { + LocalDateTime time = LocalDateTime.of(1, 1, 1, 1, 1, 1); + String customName = "name"; + PrimitiveBackupFileNameEncoder encoder = new PrimitiveBackupFileNameEncoder(); + BackupFileNameEncoder.BackupBasicInformation information = encoder.decode("kbackup-0001-01-01_01-01-01_name.zip"); + assertEquals(time, information.time); + assertEquals(customName, information.customName); + } + + @Test + public void isValid() { + PrimitiveBackupFileNameEncoder encoder = new PrimitiveBackupFileNameEncoder(); + assertTrue(encoder.isValidFileName("kbackup-0001-01-01_01-01-01_name.zip")); + assertTrue(encoder.isValidFileName("kbackup-0001-01-01_01-01-01_0001-01-01_01-01-01_name.zip")); + assertFalse(encoder.isValidFileName("kbackup-0001-01-01_01-01-01kbackup-0001-01-01_01-01-01_name.zip")); + assertFalse(encoder.isValidFileName("kbackup-0001-01-01_01-01-01_name")); + assertFalse(encoder.isValidFileName("kbackup-0001-01-01_01-01-01_name.kbi")); + assertFalse(encoder.isValidFileName("somefile")); + } } \ No newline at end of file -- cgit v1.2.3 From a28bd300a49328d04dff0a071e7cfca47cc1ed6c Mon Sep 17 00:00:00 2001 From: Keuin Date: Wed, 20 Jan 2021 17:41:35 +0800 Subject: 1.4.1: will not backup `session.lock` file. Improve test case compatibility to suit more general compile environments. --- gradle.properties | 2 +- .../java/com/keuin/kbackupfabric/util/ZipUtil.java | 20 +++++++++++++------- .../ConfiguredIncrementalBackupMethodTest.java | 4 ++-- 3 files changed, 16 insertions(+), 10 deletions(-) (limited to 'src/test/java/com/keuin/kbackupfabric/operation/backup') diff --git a/gradle.properties b/gradle.properties index 4d7ec6c..ccc429a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -6,7 +6,7 @@ minecraft_version=1.14.4 yarn_mappings=1.14.4+build.18 loader_version=0.11.0 # Mod Properties -mod_version=1.4.0 +mod_version=1.4.1 maven_group=com.keuin.kbackupfabric archives_base_name=kbackup-fabric # Dependencies diff --git a/src/main/java/com/keuin/kbackupfabric/util/ZipUtil.java b/src/main/java/com/keuin/kbackupfabric/util/ZipUtil.java index c670cf1..0fa9d77 100644 --- a/src/main/java/com/keuin/kbackupfabric/util/ZipUtil.java +++ b/src/main/java/com/keuin/kbackupfabric/util/ZipUtil.java @@ -4,7 +4,10 @@ import com.keuin.kbackupfabric.exception.ZipUtilException; import com.keuin.kbackupfabric.metadata.BackupMetadata; import java.io.*; +import java.util.Collections; import java.util.Enumeration; +import java.util.Optional; +import java.util.Set; import java.util.zip.*; public final class ZipUtil { @@ -15,9 +18,10 @@ public final class ZipUtil { * @param srcRootDir 压缩文件夹根目录的子路径 * @param file 当前递归压缩的文件或目录对象 * @param zipOutputStream 压缩文件存储对象 + * @param filesSkipping 被忽略的文件 * @throws IOException IO Error */ - private static void zip(String srcRootDir, File file, ZipOutputStream zipOutputStream) throws IOException { + private static void zip(String srcRootDir, File file, ZipOutputStream zipOutputStream, Set filesSkipping) throws IOException { if (file == null) { return; } @@ -26,7 +30,8 @@ public final class ZipUtil { return; // Reject // 如果是文件，则直接压缩该文件 - if (file.isFile()) { + boolean skipping = Optional.ofNullable(filesSkipping).orElse(Collections.emptySet()).contains(file.getName()); + if (file.isFile() && !skipping) { int count, bufferLen = 1024; byte[] data = new byte[bufferLen]; @@ -36,6 +41,8 @@ public final class ZipUtil { if (index != -1) { subPath = subPath.substring(srcRootDir.length() + File.separator.length()); } + + // 写入压缩包 ZipEntry entry = new ZipEntry(subPath); zipOutputStream.putNextEntry(entry); BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(file)); @@ -44,14 +51,13 @@ public final class ZipUtil { } inputStream.close(); zipOutputStream.closeEntry(); - } - // 如果是目录，则压缩整个目录 - else { + } else { + // 如果是目录，则压缩整个目录 // 压缩目录中的文件或子目录 File[] childFileList = file.listFiles(); if (childFileList != null) { for (File value : childFileList) - zip(srcRootDir, value, zipOutputStream); + zip(srcRootDir, value, zipOutputStream, filesSkipping); } } } @@ -127,7 +133,7 @@ public final class ZipUtil { } } //调用递归压缩方法进行目录或文件压缩 - zip(srcRootDir, srcFile, zipOutputStream); + zip(srcRootDir, srcFile, zipOutputStream, Collections.singleton("session.lock")); zipOutputStream.flush(); } finally { try { diff --git a/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java b/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java index 58e1711..30f2d44 100644 --- a/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java +++ b/src/test/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethodTest.java @@ -15,12 +15,12 @@ import static org.junit.Assert.*; public class ConfiguredIncrementalBackupMethodTest { - private final String testTempPath = "R:\\"; + private final String testTempPath = (new File("R:\\").isDirectory()) ? "R:\\" : ".\\testfile\\ConfiguredIncrementalBackupMethodTest"; private final String sourceDirectoryName = "source"; private final String destDirectoryName = "destination"; private final String indexFileName = "index"; - private final double directoryFactor = 0.03; + private final double directoryFactor = 0.02; private final double fileFactor = 0.05; private final int maxRandomFileSizeBytes = 1024 * 1024 * 16; private final Function scaleDecayFunc = (x) -> x - 1; -- cgit v1.2.3