diff options
author | Keuin <[email protected]> | 2021-01-21 01:50:08 +0800 |
---|---|---|
committer | Keuin <[email protected]> | 2021-01-21 01:50:08 +0800 |
commit | ac3b5e1476dedcefb723f19bd0fdd9a22fcb16e9 (patch) | |
tree | 75d0994f3f7aa37c3d240933f0f2d179d318e597 /src/main/java/com/keuin/kbackupfabric/util/backup | |
parent | 82e3986045ac7eaca6aaa290fb2283fd6c6c901a (diff) | |
parent | 7a5297de3467b1069fdf5e4a1b2aaf510ca35663 (diff) |
Merge remote-tracking branch 'origin/master'
# Conflicts:
# src/main/java/com/keuin/kbackupfabric/operation/backup/PrimitiveBackupMethod.java
Diffstat (limited to 'src/main/java/com/keuin/kbackupfabric/util/backup')
26 files changed, 796 insertions, 276 deletions
diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/BackupFilesystemUtil.java b/src/main/java/com/keuin/kbackupfabric/util/backup/BackupFilesystemUtil.java index 54c2f58..28ede70 100644 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/BackupFilesystemUtil.java +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/BackupFilesystemUtil.java @@ -2,10 +2,13 @@ package com.keuin.kbackupfabric.util.backup; import com.keuin.kbackupfabric.util.ReflectionUtils; import net.minecraft.server.MinecraftServer; +import net.minecraft.server.dedicated.MinecraftDedicatedServer; import net.minecraft.server.world.ThreadedAnvilChunkStorage; import net.minecraft.world.World; import java.io.File; +import java.io.IOException; +import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -14,39 +17,46 @@ import java.util.regex.Pattern; */ public final class BackupFilesystemUtil { - private static final String backupSaveDirectoryName = "backups"; + private static final String BACKUP_SAVE_DIRECTORY_NAME = "backups"; + private static final String INCREMENTAL_BASE_DIRECTORY_NAME = "incremental"; private static final String backupFileNamePrefix = "kbackup-"; + @Deprecated public static String getBackupFileNamePrefix() { return backupFileNamePrefix; } - @Deprecated - public static String getBackupFileName(String backupName) { - return backupFileNamePrefix + backupName + ".zip"; - } - @Deprecated - public static String getBackupName(String backupFileName) { - try { - if (backupFileName.matches(backupFileNamePrefix + ".+\\.zip")) - return backupFileName.substring(backupFileNamePrefix.length(), backupFileName.length() - 4); - } catch (IndexOutOfBoundsException ignored) { - } - return backupFileName; - } +// @Deprecated +// public static String getBackupName(String backupFileName) { +// try { +// if (backupFileName.matches(backupFileNamePrefix + ".+\\.zip")) +// return backupFileName.substring(backupFileNamePrefix.length(), backupFileName.length() - 4); +// } catch (IndexOutOfBoundsException ignored) { +// } +// return backupFileName; +// } - public static boolean isBackupNameValid(String backupName, MinecraftServer server) { - File backupFile = new File(getBackupSaveDirectory(server), getBackupFileName(backupName)); + public static boolean isBackupFileExists(String backupFileName, MinecraftServer server) { + File backupFile = new File(getBackupSaveDirectory(server), backupFileName); return backupFile.isFile(); } public static File getBackupSaveDirectory(MinecraftServer server) { - return new File(server.getRunDirectory(), backupSaveDirectoryName); + return new File(server.getRunDirectory(), BACKUP_SAVE_DIRECTORY_NAME); + } + + public static File getIncrementalBackupBaseDirectory(MinecraftServer server) { + return new File(server.getRunDirectory(), INCREMENTAL_BASE_DIRECTORY_NAME); } - public static String getLevelPath(MinecraftServer server) { - return (new File(server.getRunDirectory(), server.getLevelName())).getAbsolutePath(); + public static String getLevelPath(MinecraftServer server) throws IOException { + if (!(server instanceof MinecraftDedicatedServer)) + throw new IllegalStateException("This plugin is server-side only."); + String path = (new File(server.getRunDirectory().getCanonicalPath(), ((MinecraftDedicatedServer) server).getLevelName())).getAbsolutePath(); + Logger.getLogger("getLevelPath").info(String.format("Level path: %s", path)); + assert (new File(path)).exists(); + return path; } public static String getWorldDirectoryName(World world) throws NoSuchFieldException, IllegalAccessException { @@ -70,8 +80,8 @@ public final class BackupFilesystemUtil { return -1; } - public static String humanFileSize(long size) { - double fileSize = size * 1.0 / 1024 / 1024; // Default unit is MB + public static String getFriendlyFileSizeString(long sizeBytes) { + double fileSize = sizeBytes * 1.0 / 1024 / 1024; // Default unit is MB if (fileSize > 1000) //msgInfo(context, String.format("File size: %.2fGB", fileSize / 1024)); return String.format("%.2fGB", fileSize / 1024); diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/BackupType.java b/src/main/java/com/keuin/kbackupfabric/util/backup/BackupType.java deleted file mode 100644 index d02ce77..0000000 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/BackupType.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.keuin.kbackupfabric.util.backup; - -/** - * Representing the backup type. - * Should only be used in BackupFileNameBuilder and BackupFileNameFormatter - */ -@Deprecated -public enum BackupType { - - PRIMITIVE_ZIP_BACKUP("Primitive Zip Backup", "zip"), - OBJECT_TREE_BACKUP("Object Tree Backup", "incremental"); - - private final String friendlyName; // e.g. Primitive Zip Backup - private final String name; // e.g. zip - - BackupType(String friendlyName, String name) { - this.friendlyName = friendlyName; - this.name = name; - } - - /** - * Get name used in command. - * @return name (such as "zip", "incremental"). - */ - public String getName() { - return name; - } - - @Override - public String toString() { - return friendlyName; - } -} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/builder/BackupFileNameBuilder.java b/src/main/java/com/keuin/kbackupfabric/util/backup/builder/BackupFileNameBuilder.java deleted file mode 100644 index f57302c..0000000 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/builder/BackupFileNameBuilder.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.keuin.kbackupfabric.util.backup.builder; - - -import java.time.LocalDateTime; - -public interface BackupFileNameBuilder { - - static BackupFileNameBuilder primitiveZipBackup() { - return PrimitiveZipBackupFileNameBuilder.getInstance(); - } - - static BackupFileNameBuilder objectTreeBackup() { - return ObjectTreeBackupFileNameBuilder.getInstance(); - } - - /** - * Build a backup file name based on given information. - * @param time when the backup was created. - * @param backupName the custom name of this backup. Note that this should be a valid file name in current file system. - * @return the backup file name string. - */ - String build(LocalDateTime time, String backupName); - -} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/builder/ObjectTreeBackupFileNameBuilder.java b/src/main/java/com/keuin/kbackupfabric/util/backup/builder/ObjectTreeBackupFileNameBuilder.java deleted file mode 100644 index 3c15741..0000000 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/builder/ObjectTreeBackupFileNameBuilder.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.keuin.kbackupfabric.util.backup.builder; - -import com.keuin.kbackupfabric.util.backup.BackupFilesystemUtil; -import com.keuin.kbackupfabric.util.backup.BackupNameTimeFormatter; - -import java.time.LocalDateTime; - -public class ObjectTreeBackupFileNameBuilder implements BackupFileNameBuilder { - private static final ObjectTreeBackupFileNameBuilder instance = new ObjectTreeBackupFileNameBuilder(); - - public static ObjectTreeBackupFileNameBuilder getInstance() { - return instance; - } - - @Override - public String build(LocalDateTime time, String backupName) { - String timeString = BackupNameTimeFormatter.localDateTimeToString(time); - return String.format("%s%s_%s%s", BackupFilesystemUtil.getBackupFileNamePrefix(),timeString,backupName,".json"); - } -} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/builder/PrimitiveZipBackupFileNameBuilder.java b/src/main/java/com/keuin/kbackupfabric/util/backup/builder/PrimitiveZipBackupFileNameBuilder.java deleted file mode 100644 index f910c37..0000000 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/builder/PrimitiveZipBackupFileNameBuilder.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.keuin.kbackupfabric.util.backup.builder; - -import com.keuin.kbackupfabric.util.backup.BackupFilesystemUtil; -import com.keuin.kbackupfabric.util.backup.BackupNameTimeFormatter; - -import java.time.LocalDateTime; - -public class PrimitiveZipBackupFileNameBuilder implements BackupFileNameBuilder { - - private static final PrimitiveZipBackupFileNameBuilder instance = new PrimitiveZipBackupFileNameBuilder(); - - public static PrimitiveZipBackupFileNameBuilder getInstance() { - return instance; - } - - @Override - public String build(LocalDateTime time, String backupName) { - String timeString = BackupNameTimeFormatter.localDateTimeToString(time); - return String.format("%s%s_%s%s", BackupFilesystemUtil.getBackupFileNamePrefix(),timeString,backupName,".zip"); - } - -} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/formatter/BackupFileNameFormatter.java b/src/main/java/com/keuin/kbackupfabric/util/backup/formatter/BackupFileNameFormatter.java deleted file mode 100644 index a437629..0000000 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/formatter/BackupFileNameFormatter.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.keuin.kbackupfabric.util.backup.formatter; - -import com.keuin.kbackupfabric.util.backup.BackupNameTimeFormatter; - -import java.time.LocalDateTime; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public interface BackupFileNameFormatter { - - BackupFileName format(String fileName); - - class BackupFileName { - public final LocalDateTime time; - public final String name; - - public BackupFileName(LocalDateTime time, String name) { - this.time = time; - this.name = name; - } - } - - static BackupFileNameFormatter objectTreeBackup() { - return ObjectTreeBackupFileNameFormatter.getInstance(); - } - - static BackupFileNameFormatter primitiveZipBackup() { - return PrimitiveZipBackupFileNameFormatter.getInstance(); - } - -} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/formatter/ObjectTreeBackupFileNameFormatter.java b/src/main/java/com/keuin/kbackupfabric/util/backup/formatter/ObjectTreeBackupFileNameFormatter.java deleted file mode 100644 index 08805b2..0000000 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/formatter/ObjectTreeBackupFileNameFormatter.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.keuin.kbackupfabric.util.backup.formatter; - -import com.keuin.kbackupfabric.util.backup.BackupFilesystemUtil; -import com.keuin.kbackupfabric.util.backup.BackupNameTimeFormatter; -import org.spongepowered.asm.mixin.Overwrite; - -import java.time.LocalDateTime; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class ObjectTreeBackupFileNameFormatter implements BackupFileNameFormatter { - - private static final ObjectTreeBackupFileNameFormatter instance = new ObjectTreeBackupFileNameFormatter(); - - public static ObjectTreeBackupFileNameFormatter getInstance() { - return instance; - } - - @Override - public BackupFileNameFormatter.BackupFileName format(String fileName) { - LocalDateTime time = getTime(fileName); - String name = getBackupName(fileName); - return new BackupFileNameFormatter.BackupFileName(time,name); - } - - private LocalDateTime getTime(String fileName) { - Matcher matcher = Pattern.compile("[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}").matcher(fileName); - if (matcher.find()) { - String timeString = matcher.group(0); - return BackupNameTimeFormatter.timeStringToLocalDateTime(timeString); - } - return null; - } - - private String getBackupName(String backupFileName) { - try { - if (backupFileName.matches(BackupFilesystemUtil.getBackupFileNamePrefix() + ".+\\.json")) - return backupFileName.substring(BackupFilesystemUtil.getBackupFileNamePrefix().length(), backupFileName.length() - 4); - } catch (IndexOutOfBoundsException ignored) { - } - return backupFileName; - } - -} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/formatter/PrimitiveZipBackupFileNameFormatter.java b/src/main/java/com/keuin/kbackupfabric/util/backup/formatter/PrimitiveZipBackupFileNameFormatter.java deleted file mode 100644 index 2d50d17..0000000 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/formatter/PrimitiveZipBackupFileNameFormatter.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.keuin.kbackupfabric.util.backup.formatter; - -import com.keuin.kbackupfabric.util.backup.BackupFilesystemUtil; -import com.keuin.kbackupfabric.util.backup.BackupNameTimeFormatter; - -import java.time.LocalDateTime; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class PrimitiveZipBackupFileNameFormatter implements BackupFileNameFormatter { - - private static final PrimitiveZipBackupFileNameFormatter instance = new PrimitiveZipBackupFileNameFormatter(); - - public static PrimitiveZipBackupFileNameFormatter getInstance() { - return instance; - } - - @Override - public BackupFileNameFormatter.BackupFileName format(String fileName) { - LocalDateTime time = getTime(fileName); - String name = getBackupName(fileName); - return new BackupFileNameFormatter.BackupFileName(time,name); - } - - private LocalDateTime getTime(String fileName) { - Matcher matcher = Pattern.compile("[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}").matcher(fileName); - if (matcher.find()) { - String timeString = matcher.group(0); - return BackupNameTimeFormatter.timeStringToLocalDateTime(timeString); - } - return null; - } - - private String getBackupName(String backupFileName) { - try { - if (backupFileName.matches(BackupFilesystemUtil.getBackupFileNamePrefix() + ".+\\.zip")) - return backupFileName.substring(BackupFilesystemUtil.getBackupFileNamePrefix().length(), backupFileName.length() - 4); - } catch (IndexOutOfBoundsException ignored) { - } - return backupFileName; - } - -} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectCollection.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectCollection.java new file mode 100644 index 0000000..16d95e6 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectCollection.java @@ -0,0 +1,65 @@ +package com.keuin.kbackupfabric.util.backup.incremental; + +import java.io.Serializable; +import java.util.*; + +public class ObjectCollection implements Serializable { + private final String name; + private final Map<String, ObjectElement> elements; + private final Map<String, ObjectCollection> subCollections; + + ObjectCollection(String name, Set<ObjectElement> elements, Map<String, ObjectCollection> subCollections) { + this.name = Objects.requireNonNull(name); + this.elements = new HashMap<>(); + for (ObjectElement e : elements) { + Objects.requireNonNull(e); + if (this.elements.put(e.getName(), e) != null) { + throw new IllegalStateException("elements conflict with the same name"); + } + } + this.subCollections = new HashMap<>(Objects.requireNonNull(subCollections)); + } + + public String getName() { + return name; + } + + public Set<ObjectElement> getElementSet() { + return new HashSet<>(elements.values()); + } + + public Map<String, ObjectElement> getElementMap() { + return Collections.unmodifiableMap(elements); + } + + public ObjectElement getElement(String name) { + return elements.get(name); + } + + public Set<ObjectCollection> getSubCollectionSet() { + return new HashSet<>(subCollections.values()); + } + + public Map<String, ObjectCollection> getSubCollectionMap() { + return Collections.unmodifiableMap(subCollections); + } + + public ObjectCollection getSubCollection(String name) { + return subCollections.get(name); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ObjectCollection that = (ObjectCollection) o; + return name.equals(that.name) && + elements.equals(that.elements) && + subCollections.equals(that.subCollections); + } + + @Override + public int hashCode() { + return Objects.hash(name, elements, subCollections); + } +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectCollectionFactory.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectCollectionFactory.java new file mode 100644 index 0000000..2f3761c --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectCollectionFactory.java @@ -0,0 +1,55 @@ +package com.keuin.kbackupfabric.util.backup.incremental; + +import com.keuin.kbackupfabric.util.PrintUtil; +import com.keuin.kbackupfabric.util.backup.incremental.identifier.FileIdentifierProvider; +import com.keuin.kbackupfabric.util.backup.incremental.identifier.ObjectIdentifier; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +/** + * Incremental backup is implemented as git-like file collection. + * Files are called `objects`, the collection contains all files distinguished by their + * identifiers. Usually, identifier is the combination of hash and other short information (such as size and another hash). + * The identifier should use hashes that are strong enough, to prevent possible collisions. + */ +public class ObjectCollectionFactory<T extends ObjectIdentifier> { + + private final FileIdentifierProvider<T> identifierFactory; + + public ObjectCollectionFactory(FileIdentifierProvider<T> identifierFactory) { + this.identifierFactory = identifierFactory; + } + + public ObjectCollection fromDirectory(File directory, Set<String> ignoredFiles) throws IOException { + final Set<ObjectElement> subFiles = new HashSet<>(); + final Map<String, ObjectCollection> subCollections = new HashMap<>(); + + if (!Objects.requireNonNull(directory).isDirectory()) + throw new IllegalArgumentException("given file is not a directory"); + + for (Iterator<Path> iter = Files.walk(directory.toPath(), 1).iterator(); iter.hasNext(); ) { + Path path = iter.next(); + if (Files.isSameFile(path, directory.toPath())) + continue; + File file = path.toFile(); + if (file.isDirectory()) { + subCollections.put(file.getName(), fromDirectory(file, ignoredFiles)); + } else if (!ignoredFiles.contains(file.getName())) { + subFiles.add(new ObjectElement(file.getName(), identifierFactory.fromFile(file))); + } else { + PrintUtil.info(String.format("Skipping file %s.", file.getName())); + } + } + + return new ObjectCollection(directory.getName(), subFiles, subCollections); + } + + public ObjectCollection fromDirectory(File directory) throws IOException { + return fromDirectory(directory, Collections.emptySet()); + } + +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectCollectionSerializer.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectCollectionSerializer.java new file mode 100644 index 0000000..6f9b792 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectCollectionSerializer.java @@ -0,0 +1,33 @@ +package com.keuin.kbackupfabric.util.backup.incremental; + +import java.io.*; +import java.util.Objects; + +/** + * Serialize and deserialize ObjectCollection from/to the disk file. + */ +public class ObjectCollectionSerializer { + public static ObjectCollection fromFile(File file) throws IOException { + Objects.requireNonNull(file); + ObjectCollection collection; + try (FileInputStream fileInputStream = new FileInputStream(file)) { + try (ObjectInputStream objectInputStream = new ObjectInputStream(fileInputStream)) { + collection = (ObjectCollection) objectInputStream.readObject(); + } catch (ClassNotFoundException ignored) { + // this should not happen + return null; + } + } + return collection; + } + + public static void toFile(ObjectCollection collection, File file) throws IOException { + Objects.requireNonNull(collection); + Objects.requireNonNull(file); + try (FileOutputStream fileOutputStream = new FileOutputStream(file)) { + try (ObjectOutputStream objectOutputStream = new ObjectOutputStream(fileOutputStream)) { + objectOutputStream.writeObject(collection); + } + } + } +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectElement.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectElement.java new file mode 100644 index 0000000..cbb886e --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/ObjectElement.java @@ -0,0 +1,60 @@ +package com.keuin.kbackupfabric.util.backup.incremental; + +import com.keuin.kbackupfabric.util.backup.incremental.identifier.ObjectIdentifier; + +import java.io.Serializable; +import java.util.Objects; + +/** + * Representing a file in a ObjectCollection. + * Immutable. + */ +public class ObjectElement implements Serializable { + private final String name; + private final ObjectIdentifier identifier; + + public ObjectElement(String name, ObjectIdentifier identifier) { + Objects.requireNonNull(name); + Objects.requireNonNull(identifier); + this.name = name; + this.identifier = identifier; + } + + /** + * Get file name. + * @return the file name. + */ + public String getName() { + return name; + } + + /** + * Get file identifier, which is considered to be different between files with different contents. + * @return the identifier. + */ + public ObjectIdentifier getIdentifier() { + return identifier; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ObjectElement that = (ObjectElement) o; + return name.equals(that.name) && + identifier.equals(that.identifier); + } + + @Override + public int hashCode() { + return Objects.hash(name, identifier); + } + + @Override + public String toString() { + return "ObjectElement{" + + "name='" + name + '\'' + + ", identifier=" + identifier + + '}'; + } +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/FileIdentifierProvider.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/FileIdentifierProvider.java new file mode 100644 index 0000000..3fbe284 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/FileIdentifierProvider.java @@ -0,0 +1,15 @@ +package com.keuin.kbackupfabric.util.backup.incremental.identifier; + +import java.io.File; +import java.io.IOException; + +public interface FileIdentifierProvider<T extends ObjectIdentifier> { + /** + * Generate file identifier from a random file. The file is not necessarily in the object base. + * + * @param file the file. + * @return the file identifier. + * @throws IOException when an I/O error occurs. + */ + T fromFile(File file) throws IOException; +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/ObjectIdentifier.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/ObjectIdentifier.java new file mode 100644 index 0000000..aece07d --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/ObjectIdentifier.java @@ -0,0 +1,13 @@ +package com.keuin.kbackupfabric.util.backup.incremental.identifier; + +import java.io.Serializable; + +/** + * The identifier distinguishing files in the object collection. + * It should be based on cryptographic hash function in order to prevent possible attacks to the backup system. + * All identifiers should be immutable and implement their own equals method. + * Immutable. + */ +public interface ObjectIdentifier extends Serializable { + String getIdentification(); +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/Sha256Identifier.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/Sha256Identifier.java new file mode 100644 index 0000000..c1c87e1 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/Sha256Identifier.java @@ -0,0 +1,88 @@ +package com.keuin.kbackupfabric.util.backup.incremental.identifier; + +import com.keuin.kbackupfabric.util.BytesUtil; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Objects; + +/** + * Identifier based on sha256. + * Immutable. + */ +public class Sha256Identifier extends SingleHashIdentifier { + + private static final int SHA256_LENGTH = 32; + private static final Sha256Identifier DUMMY = new Sha256Identifier(new byte[SHA256_LENGTH]); // only for using its hash method + private static final FileIdentifierProvider<Sha256Identifier> factory = Sha256Identifier::fromFile; + private static final String marker = "S2"; + + public static Sha256Identifier fromFile(File file) throws IOException { + if (!file.isFile()) { + throw new IllegalArgumentException("file is not a file"); + } + return new Sha256Identifier(DUMMY.hash(file)); + } + + /** + * Load sha-256 from a named file. Only used in StorageObjectLoader. + * + * @param fileName the file name. + * @return identifier. + */ + static Sha256Identifier fromFileName(String fileName) { + if (!fileName.matches(marker + "-[0-9A-Fa-f]{32}")) + return null; + String hexString = fileName.substring(marker.length() + 1); + return new Sha256Identifier(BytesUtil.hexToBytes(hexString)); + } + + public static FileIdentifierProvider<Sha256Identifier> getFactory() { + return factory; + } + + protected Sha256Identifier(byte[] hash) { + super(hash, marker); + Objects.requireNonNull(hash); + if (hash.length != SHA256_LENGTH) { + throw new IllegalStateException(String.format("SHA256 must be %d bytes", SHA256_LENGTH)); + } + } + + @Override + protected byte[] hash(File file) throws IOException { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + + try (FileInputStream inputStream = new FileInputStream(file)) { + // This does not work. I don't know why +// FileChannel channel = inputStream.getChannel(); +// ByteBuffer buffer = ByteBuffer.allocate(128); +// int readLength; +// while ((readLength = channel.read(buffer)) > 0) +// digest.update(buffer); + + // This also works, without warnings + byte[] readBuffer = new byte[1024 * 1024]; + int readLength; + while ((readLength = inputStream.read(readBuffer)) > 0) + digest.update(readBuffer, 0, readLength); + + // The below lines also works, but the IDE will complain about the while loop +// DigestInputStream digestInputStream = new DigestInputStream(inputStream, digest); +// while(digestInputStream.read() > 0) +// ; + + return digest.digest(); + } + + } catch (NoSuchAlgorithmException ignored) { + // this shouldn't happen + return new byte[SHA256_LENGTH]; + } + } + +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/SingleHashIdentifier.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/SingleHashIdentifier.java new file mode 100644 index 0000000..0f62f2b --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/SingleHashIdentifier.java @@ -0,0 +1,53 @@ +package com.keuin.kbackupfabric.util.backup.incremental.identifier; + +import com.keuin.kbackupfabric.util.BytesUtil; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +/** + * A simple identifier based on a single hash function. + * Immutable. + */ +public abstract class SingleHashIdentifier implements ObjectIdentifier { + + private final byte[] hash; + private final String type; + + protected SingleHashIdentifier(byte[] hash, String type) { + Objects.requireNonNull(hash); + Objects.requireNonNull(type); + this.hash = Arrays.copyOf(hash, hash.length); + this.type = type; + } + + /** + * The hash function. + * + * @param file the file to be hashed. + * @return the hash bytes. + */ + protected abstract byte[] hash(File file) throws IOException; + + @Override + public String getIdentification() { + return type + "-" + BytesUtil.bytesToHex(hash); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof SingleHashIdentifier)) { + return false; + } + return Arrays.equals(hash, ((SingleHashIdentifier) obj).hash); + } + + @Override + public int hashCode() { + int result = Objects.hash(type); + result = 31 * result + Arrays.hashCode(hash); + return result; + } +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/StorageObjectLoader.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/StorageObjectLoader.java new file mode 100644 index 0000000..96bc295 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/identifier/StorageObjectLoader.java @@ -0,0 +1,25 @@ +package com.keuin.kbackupfabric.util.backup.incremental.identifier; + +import java.io.File; +import java.util.Objects; + +public class StorageObjectLoader { + /** + * Get identifier from storage file. + * + * @param file storage file. + * @return identifier. If failed, return null. + */ + public static ObjectIdentifier asIdentifier(File file) { + Objects.requireNonNull(file); + String fileName = file.getName(); + ObjectIdentifier identifier; + + identifier = Sha256Identifier.fromFileName(fileName); + if (identifier != null) + return identifier; + + // Add more identifiers. + return null; + } +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/manager/IncrementalBackupStorageManager.java b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/manager/IncrementalBackupStorageManager.java new file mode 100644 index 0000000..6fd339b --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/incremental/manager/IncrementalBackupStorageManager.java @@ -0,0 +1,188 @@ +package com.keuin.kbackupfabric.util.backup.incremental.manager; + +import com.keuin.kbackupfabric.util.PrintUtil; +import com.keuin.kbackupfabric.util.backup.incremental.ObjectCollection; +import com.keuin.kbackupfabric.util.backup.incremental.ObjectElement; +import com.keuin.kbackupfabric.util.backup.incremental.identifier.ObjectIdentifier; +import com.keuin.kbackupfabric.util.backup.incremental.identifier.StorageObjectLoader; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; + +import static org.apache.commons.io.FileUtils.forceDelete; + +public class IncrementalBackupStorageManager { + + private final Path backupStorageBase; + private final Map<ObjectIdentifier, File> map = new HashMap<>(); + private boolean loaded = false; + + public IncrementalBackupStorageManager(Path backupStorageBase) { + this.backupStorageBase = backupStorageBase; + } + + /** + * Add a object collection to storage base. + * @param collection the collection. + * @return objects copied to the base. + * @throws IOException I/O Error. + */ + public int addObjectCollection(ObjectCollection collection, File collectionBasePath) throws IOException { + if (!backupStorageBase.toFile().isDirectory()) { + if (!backupStorageBase.toFile().mkdirs()) + throw new IOException("Backup storage base directory does not exist, and failed to create it."); + } + Objects.requireNonNull(collection); + Objects.requireNonNull(collectionBasePath); + + int copyCount = 0; + + // copy sub files + for (Map.Entry<String, ObjectElement> entry : collection.getElementMap().entrySet()) { + File copyDestination = new File(backupStorageBase.toFile(), entry.getValue().getIdentifier().getIdentification()); + if (!baseContainsObject(entry.getValue())) { + // element does not exist. copy. + Files.copy(Paths.get(collectionBasePath.getAbsolutePath(), entry.getKey()), copyDestination.toPath()); + ++copyCount; + } + } + + //copy sub dirs recursively + for (Map.Entry<String, ObjectCollection> entry : collection.getSubCollectionMap().entrySet()) { + File newBase = new File(collectionBasePath, entry.getKey()); + copyCount += addObjectCollection(entry.getValue(), newBase); + } + + return copyCount; + } + + /** + * Restore an object collection from the storage base. i.e., restore the save from backup storage. + * @param collection the collection to be restored. + * @param collectionBasePath save path of the collection. + * @return objects restored from the base. + * @throws IOException I/O Error. + */ + public int restoreObjectCollection(ObjectCollection collection, File collectionBasePath) throws IOException { + Objects.requireNonNull(collection); + Objects.requireNonNull(collectionBasePath); + + int copyCount = 0; + + // touch directory + if (!collectionBasePath.exists()) { + int retryCounter = 0; + boolean success = false; + while (retryCounter++ < 5) { + if (collectionBasePath.mkdirs()) { + success = true; + break; + } + } + if (!success) { + throw new IOException("Failed to create directory " + collectionBasePath.getAbsolutePath()); + } + } + + // copy sub files + for (Map.Entry<String, ObjectElement> entry : collection.getElementMap().entrySet()) { + File copySource = new File(backupStorageBase.toFile(), entry.getValue().getIdentifier().getIdentification()); + File copyTarget = new File(collectionBasePath.getAbsolutePath(), entry.getKey()); + + if (!baseContainsObject(entry.getValue())) { + throw new IOException(String.format("File %s does not exist in the base.", copySource.getName())); + } + if (copyTarget.exists()) { + boolean successDeleting = false; + for (int i = 0; i < 5; ++i) { + try { + forceDelete(copyTarget); + successDeleting = true; + break; + } catch (FileNotFoundException ignored) { + break; + } catch (IOException e) { + PrintUtil.error(String.format("Failed to delete file %s, retry.", copyTarget.getName())); + } + } + if (!successDeleting) { + String msg = String.format("Failed to delete file %s.", copyTarget.getName()); + PrintUtil.error(msg); + throw new IOException(msg); + } + } + + Files.copy(copySource.toPath(), copyTarget.toPath()); + ++copyCount; + } + + //copy sub dirs recursively + for (Map.Entry<String, ObjectCollection> entry : collection.getSubCollectionMap().entrySet()) { + File newBase = new File(collectionBasePath, entry.getKey()); + copyCount += restoreObjectCollection(entry.getValue(), newBase); + } + + return copyCount; + } + + public int cleanUnusedObjects(Iterable<ObjectCollection> collectionIterable) { + // construct object list in memory + Set<String> objects = new HashSet<>(); +// backupStorageBase + + for (ObjectCollection collection : collectionIterable) { + for (ObjectElement ele : collection.getElementMap().values()) { + + } + } + throw new RuntimeException("not impl"); + } + + /** + * Check all objects, return unused ones. + * + * @return the unused ones. + */ + private Map<ObjectIdentifier, File> markUnusedObjects() { + throw new RuntimeException("not impl"); + } + + /** + * Check if the backup base contains given element. + * + * @param objectElement the element. + * @return true or false. + */ + private boolean baseContainsObject(ObjectElement objectElement) { + // This may be extended to use more variants of hash functions and combinations of other attributes (such as file size) + return (new File(backupStorageBase.toFile(), objectElement.getIdentifier().getIdentification())).exists(); + } + + private void lazyLoadStorage() throws IOException { + if (!loaded) { + loadStorage(); + loaded = true; + } + } + + private synchronized void loadStorage() throws IOException { + map.clear(); + Files.walk(backupStorageBase, 1).forEach(path -> { + File file = path.toFile(); + ObjectIdentifier identifier = StorageObjectLoader.asIdentifier(file); + if (identifier == null) { + map.clear(); + throw new IllegalStateException(String.format( + "Bad storage object %s: cannot recognize identifier.", file.getName() + )); + } + map.put(identifier, file); + }); + } + +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/name/BackupFileNameEncoder.java b/src/main/java/com/keuin/kbackupfabric/util/backup/name/BackupFileNameEncoder.java new file mode 100644 index 0000000..8ebc7ff --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/name/BackupFileNameEncoder.java @@ -0,0 +1,64 @@ +package com.keuin.kbackupfabric.util.backup.name; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +/** + * Encode and decode backup file name for a specific backup type. + */ +public interface BackupFileNameEncoder { + + /** + * Construct full backup file name from custom name and creation time. + * @param customName the custom name. If the custom name contains invalid chars, an exception will be thrown. + * @param time the creation time. + * @return the file name. + */ + String encode(String customName, LocalDateTime time); + + /** + * Extract custom and backup time from backup file name. + * + * @param fileName the backup file name. + * @return the information. If the given file name is invalid, return null. + */ + BackupBasicInformation decode(String fileName); + + default boolean isValidFileName(String fileName) { + return decode(fileName) != null; + } + + /** + * Check if the given string is a valid custom backup name. + * + * @param customName the custom backup name. + * @return if the name is valid. + */ + default boolean isValidCustomName(String customName) { + final char[] ILLEGAL_CHARACTERS = {'/', '\n', '\r', '\t', '\0', '\f', '`', '?', '*', '\\', '<', '>', '|', '\"', ':'}; + for (char c : ILLEGAL_CHARACTERS) { + if (customName.contains(String.valueOf(c))) { + return false; + } + } + return true; + } + + class BackupBasicInformation { + + public final String customName; + public final LocalDateTime time; + + private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm.ss"); + + protected BackupBasicInformation(String customName, LocalDateTime time) { + this.customName = customName; + this.time = time; + } + + @Override + public String toString() { + return String.format("%s, %s", customName, time.format(formatter)); + } + } +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/name/IncrementalBackupFileNameEncoder.java b/src/main/java/com/keuin/kbackupfabric/util/backup/name/IncrementalBackupFileNameEncoder.java new file mode 100644 index 0000000..926f47c --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/name/IncrementalBackupFileNameEncoder.java @@ -0,0 +1,37 @@ +package com.keuin.kbackupfabric.util.backup.name; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class IncrementalBackupFileNameEncoder implements BackupFileNameEncoder { + private static final String backupFileNamePrefix = "incremental"; + private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss"); + + // TODO: make this private and use singleton pattern + public IncrementalBackupFileNameEncoder() { + } + + @Override + public String encode(String customName, LocalDateTime time) { + if (!isValidCustomName(customName)) + throw new IllegalArgumentException("Invalid custom name"); + String timeString = time.format(formatter); + return backupFileNamePrefix + "-" + timeString + "_" + customName + ".kbi"; + } + + @Override + public BackupFileNameEncoder.BackupBasicInformation decode(String fileName) { + Pattern pattern = Pattern.compile( + "^" + backupFileNamePrefix + "-" + "([0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2})_(.+)\\.kbi" + "$" + ); + Matcher matcher = pattern.matcher(fileName); + if (matcher.find()) { + String timeString = matcher.group(1); + String customName = matcher.group(2); + return new BackupFileNameEncoder.BackupBasicInformation(customName, LocalDateTime.parse(timeString, formatter)); + } + return null; + } +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/name/PrimitiveBackupFileNameEncoder.java b/src/main/java/com/keuin/kbackupfabric/util/backup/name/PrimitiveBackupFileNameEncoder.java new file mode 100644 index 0000000..ef15ae7 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/name/PrimitiveBackupFileNameEncoder.java @@ -0,0 +1,33 @@ +package com.keuin.kbackupfabric.util.backup.name; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class PrimitiveBackupFileNameEncoder implements BackupFileNameEncoder { + private static final String backupFileNamePrefix = "kbackup"; + private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss"); + + @Override + public String encode(String customName, LocalDateTime time) { + if (!isValidCustomName(customName)) + throw new IllegalArgumentException("Invalid custom name"); + String timeString = time.format(formatter); + return backupFileNamePrefix + "-" + timeString + "_" + customName + ".zip"; + } + + @Override + public BackupBasicInformation decode(String fileName) { + Pattern pattern = Pattern.compile( + "^" + backupFileNamePrefix + "-" + "([0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2})_(.+)\\.zip" + "$" + ); + Matcher matcher = pattern.matcher(fileName); + if (matcher.find()) { + String timeString = matcher.group(1); + String customName = matcher.group(2); + return new BackupBasicInformation(customName, LocalDateTime.parse(timeString, formatter)); + } + return null; + } +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/provider/AvailableBackupProvider.java b/src/main/java/com/keuin/kbackupfabric/util/backup/provider/AvailableBackupProvider.java new file mode 100644 index 0000000..caa0e84 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/provider/AvailableBackupProvider.java @@ -0,0 +1,8 @@ +package com.keuin.kbackupfabric.util.backup.provider; + +/** + * List all backup in disk. Provide their basic information as soon as possible. + */ +public class AvailableBackupProvider { + // TODO: remove obsolete impl. in command user interface. Use this instead. +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/provider/IncrementalBackupInformation.java b/src/main/java/com/keuin/kbackupfabric/util/backup/provider/IncrementalBackupInformation.java new file mode 100644 index 0000000..861d210 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/provider/IncrementalBackupInformation.java @@ -0,0 +1,13 @@ +package com.keuin.kbackupfabric.util.backup.provider; + +import com.keuin.kbackupfabric.util.backup.name.BackupFileNameEncoder; + +import java.time.LocalDateTime; + +public class IncrementalBackupInformation extends BackupFileNameEncoder.BackupBasicInformation { + // TODO: show total size for incremental backup + + public IncrementalBackupInformation(String customName, LocalDateTime time) { + super(customName, time); + } +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/provider/PrimitiveBackupInformation.java b/src/main/java/com/keuin/kbackupfabric/util/backup/provider/PrimitiveBackupInformation.java new file mode 100644 index 0000000..d3d2db8 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/provider/PrimitiveBackupInformation.java @@ -0,0 +1,14 @@ +package com.keuin.kbackupfabric.util.backup.provider; + +import com.keuin.kbackupfabric.util.backup.name.BackupFileNameEncoder; + +import java.time.LocalDateTime; + +public class PrimitiveBackupInformation extends BackupFileNameEncoder.BackupBasicInformation { + public final long sizeBytes; + + public PrimitiveBackupInformation(String customName, LocalDateTime time, long sizeBytes) { + super(customName, time); + this.sizeBytes = sizeBytes; + } +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/suggestion/BackupMethodSuggestionProvider.java b/src/main/java/com/keuin/kbackupfabric/util/backup/suggestion/BackupMethodSuggestionProvider.java deleted file mode 100644 index 320d9bf..0000000 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/suggestion/BackupMethodSuggestionProvider.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.keuin.kbackupfabric.util.backup.suggestion; - -import com.keuin.kbackupfabric.util.backup.BackupType; -import com.mojang.brigadier.suggestion.SuggestionProvider; -import com.mojang.brigadier.suggestion.Suggestions; -import com.mojang.brigadier.suggestion.SuggestionsBuilder; -import net.minecraft.server.command.ServerCommandSource; - -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Locale; -import java.util.concurrent.CompletableFuture; - -public class BackupMethodSuggestionProvider { - - private static final List<String> suggestions = Arrays.asList( - BackupType.OBJECT_TREE_BACKUP.getName(), - BackupType.PRIMITIVE_ZIP_BACKUP.getName() - ); // All backup methods - - public static SuggestionProvider<ServerCommandSource> getProvider() { - return (context, builder) -> getCompletableFuture(builder); - } - - private static CompletableFuture<Suggestions> getCompletableFuture(SuggestionsBuilder builder) { - String remaining = builder.getRemaining().toLowerCase(Locale.ROOT); - for (String string : suggestions) { // Iterate through the supplied list - if (string.toLowerCase(Locale.ROOT).startsWith(remaining)) { - builder.suggest(string); // Add every single entry to suggestions list. - } - } - return builder.buildFuture(); // Create the CompletableFuture containing all the suggestions - } - -} diff --git a/src/main/java/com/keuin/kbackupfabric/util/backup/suggestion/BackupNameSuggestionProvider.java b/src/main/java/com/keuin/kbackupfabric/util/backup/suggestion/BackupNameSuggestionProvider.java index f6f4056..01152c2 100644 --- a/src/main/java/com/keuin/kbackupfabric/util/backup/suggestion/BackupNameSuggestionProvider.java +++ b/src/main/java/com/keuin/kbackupfabric/util/backup/suggestion/BackupNameSuggestionProvider.java @@ -1,6 +1,5 @@ package com.keuin.kbackupfabric.util.backup.suggestion; -import com.keuin.kbackupfabric.util.backup.BackupFilesystemUtil; import com.mojang.brigadier.suggestion.SuggestionProvider; import com.mojang.brigadier.suggestion.Suggestions; import com.mojang.brigadier.suggestion.SuggestionsBuilder; @@ -38,7 +37,7 @@ public class BackupNameSuggestionProvider { if (files == null) return; for (File f : files) - candidateCacheList.add(BackupFilesystemUtil.getBackupName(f.getName())); + candidateCacheList.add(f.getName()); cacheUpdateTime = System.currentTimeMillis(); } catch (NullPointerException ignored) { } |