diff options
Diffstat (limited to 'src/main')
5 files changed, 156 insertions, 52 deletions
diff --git a/src/main/java/com/keuin/kbackupfabric/KBCommands.java b/src/main/java/com/keuin/kbackupfabric/KBCommands.java index d9213c3..9c887fd 100644 --- a/src/main/java/com/keuin/kbackupfabric/KBCommands.java +++ b/src/main/java/com/keuin/kbackupfabric/KBCommands.java @@ -90,6 +90,7 @@ public final class KBCommands { // TODO: Show real name and size and etc info for incremental backup // TODO: Show concrete info from metadata for `.zip` backup MinecraftServer server = context.getSource().getMinecraftServer(); + // TODO: refactor this to use {@link ObjectCollectionSerializer#fromDirectory} File[] files = getBackupSaveDirectory(server).listFiles( (dir, name) -> dir.isDirectory() && (name.toLowerCase().endsWith(".zip") && name.toLowerCase().startsWith(getBackupFileNamePrefix()) @@ -100,6 +101,7 @@ public final class KBCommands { Objects.requireNonNull(file); if (file.getName().toLowerCase().endsWith(".zip")) return getPrimitiveBackupInformationString(file.getName(), file.length()); + // TODO: refactor this to use {@link ObjectCollectionSerializer#fromDirectory} else if (file.getName().toLowerCase().endsWith(".kbi")) return getIncrementalBackupInformationString(file); return file.getName(); diff --git a/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionIterator.java b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionIterator.java new file mode 100644 index 0000000..248d36d --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionIterator.java @@ -0,0 +1,50 @@ +package com.keuin.kbackupfabric.backup.incremental; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.NoSuchElementException; + +public class ObjectCollectionIterator implements Iterator<ObjectElement> { + + // TODO: test this + + private Iterator<ObjectElement> currentIterator; + private final List<ObjectCollection2> cols = new LinkedList<>(); + + public ObjectCollectionIterator(ObjectCollection2 collection) { + cols.addAll(collection.getSubCollectionSet()); + currentIterator = collection.getElementSet().iterator(); + } + + @Override + public boolean hasNext() { + if (currentIterator != null) { + if (currentIterator.hasNext()) + return true; + else { + currentIterator = null; + return hasNext(); + } + } else { + if (cols.isEmpty()) + return false; + else { + ObjectCollection2 consumedCollection = cols.remove(0); + cols.addAll(consumedCollection.getSubCollectionSet()); + currentIterator = consumedCollection.getElementSet().iterator(); + return hasNext(); + } + } + } + + @Override + public ObjectElement next() { + if (hasNext()) { + return currentIterator.next(); + } else { + throw new NoSuchElementException(); + } + } + +} diff --git a/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializer.java b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializer.java index f663f20..fa411a0 100644 --- a/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializer.java +++ b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializer.java @@ -1,6 +1,11 @@ package com.keuin.kbackupfabric.backup.incremental; +import org.jetbrains.annotations.NotNull; + import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; import java.util.Objects; /** @@ -39,4 +44,40 @@ public class ObjectCollectionSerializer { } } } + + public static Iterable<ObjectCollection2> fromDirectory(File directory) throws IOException { + + if (!directory.isDirectory()) { + throw new IllegalArgumentException("Given directory is invalid."); + } + return new Iterable<ObjectCollection2>() { + private final Iterator<ObjectCollection2> iter = new Iterator<ObjectCollection2>() { + private final Iterator<Path> i = Files.walk(directory.toPath(), 1).filter(p -> { + File f = p.toFile(); + return f.isFile() && f.getName().endsWith(".kbi"); + }).iterator(); + + @Override + public boolean hasNext() { + return i.hasNext(); + } + + @Override + public ObjectCollection2 next() { + try { + return fromFile(i.next().toFile()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }; + + @NotNull + @Override + public Iterator<ObjectCollection2> iterator() { + return iter; + } + }; + + } } diff --git a/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java b/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java index be01966..ad7287f 100644 --- a/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java +++ b/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java @@ -1,9 +1,9 @@ package com.keuin.kbackupfabric.backup.incremental.manager; import com.keuin.kbackupfabric.backup.incremental.ObjectCollection2; +import com.keuin.kbackupfabric.backup.incremental.ObjectCollectionIterator; import com.keuin.kbackupfabric.backup.incremental.ObjectElement; import com.keuin.kbackupfabric.backup.incremental.identifier.ObjectIdentifier; -import com.keuin.kbackupfabric.backup.incremental.identifier.StorageObjectLoader; import com.keuin.kbackupfabric.util.FilesystemUtil; import com.keuin.kbackupfabric.util.PrintUtil; import org.jetbrains.annotations.Nullable; @@ -14,6 +14,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; +import java.util.logging.Logger; import static org.apache.commons.io.FileUtils.forceDelete; @@ -23,6 +24,8 @@ public class IncrementalBackupStorageManager { private final Map<ObjectIdentifier, File> map = new HashMap<>(); private boolean loaded = false; + private final Logger LOGGER = Logger.getLogger(IncrementalBackupStorageManager.class.getName()); + public IncrementalBackupStorageManager(Path backupStorageBase) { this.backupStorageBase = backupStorageBase; } @@ -32,7 +35,7 @@ public class IncrementalBackupStorageManager { * * @param collection the collection. * @return objects copied to the base. - * @throws IOException I/O Error. + * @throws IOException I/O error. */ public @Nullable IncCopyResult addObjectCollection(ObjectCollection2 collection, File collectionBasePath) throws IOException { @@ -71,6 +74,37 @@ public class IncrementalBackupStorageManager { } /** + * Delete all files in the specific collection, from the storage base. + * + * @param collection the collection containing files to be deleted. + * @param collectionBasePath the collection base path. + * @throws IOException I/O error. + */ + public void deleteObjectCollection(ObjectCollection2 collection, File collectionBasePath) throws IOException { + deleteObjectCollection(collection, collectionBasePath, Collections.emptySet()); + } + + /** + * Delete a collection from the storage base, optionally preserving files used by other backups. + * + * @param collection the collection containing files to be deleted. + * @param collectionBasePath the collection base path. + * @param otherExistingCollections other collections (not to be deleted) in this base. Files exist in these collections will not be deleted. + */ + public void deleteObjectCollection(ObjectCollection2 collection, File collectionBasePath, + Iterable<ObjectCollection2> otherExistingCollections) { + Iterator<ObjectElement> iter = new ObjectCollectionIterator(collection); + Set<ObjectElement> unusedElementSet = new HashSet<>(); + iter.forEachRemaining(unusedElementSet::add); + otherExistingCollections.forEach(col -> new ObjectCollectionIterator(col).forEachRemaining(unusedElementSet::remove)); + unusedElementSet.forEach(ele -> { + File file = new File(backupStorageBase.toFile(), ele.getIdentifier().getIdentification()); + if (!file.delete()) + LOGGER.warning("Failed to delete unused file " + file.getName()); + }); + } + + /** * Restore an object collection from the storage base. i.e., restore the save from backup storage. * * @param collection the collection to be restored. @@ -140,28 +174,6 @@ public class IncrementalBackupStorageManager { return copyCount; } - public int cleanUnusedObjects(Iterable<ObjectCollection2> collectionIterable) { - // construct object list in memory - Set<String> objects = new HashSet<>(); -// backupStorageBase - - for (ObjectCollection2 collection : collectionIterable) { - for (ObjectElement ele : collection.getElementMap().values()) { - - } - } - throw new RuntimeException("not impl"); - } - - /** - * Check all objects, return unused ones. - * - * @return the unused ones. - */ - private Map<ObjectIdentifier, File> markUnusedObjects() { - throw new RuntimeException("not impl"); - } - /** * Check if the backup base contains given element. * @@ -174,25 +186,4 @@ public class IncrementalBackupStorageManager { return (new File(backupStorageBase.toFile(), objectElement.getIdentifier().getIdentification())).exists(); } - private void lazyLoadStorage() throws IOException { - if (!loaded) { - loadStorage(); - loaded = true; - } - } - - private synchronized void loadStorage() throws IOException { - map.clear(); - Files.walk(backupStorageBase, 1).forEach(path -> { - File file = path.toFile(); - ObjectIdentifier identifier = StorageObjectLoader.asIdentifier(file); - if (identifier == null) { - map.clear(); - throw new IllegalStateException(String.format( - "Bad storage object %s: cannot recognize identifier.", file.getName() - )); - } - map.put(identifier, file); - }); - } } diff --git a/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java b/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java index 2a9cbc8..ffcc000 100644 --- a/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java +++ b/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java @@ -2,6 +2,7 @@ package com.keuin.kbackupfabric.operation.backup.method; import com.keuin.kbackupfabric.backup.incremental.ObjectCollection2; import com.keuin.kbackupfabric.backup.incremental.ObjectCollectionFactory; +import com.keuin.kbackupfabric.backup.incremental.ObjectCollectionSerializer; import com.keuin.kbackupfabric.backup.incremental.identifier.Sha256Identifier; import com.keuin.kbackupfabric.backup.incremental.manager.IncCopyResult; import com.keuin.kbackupfabric.backup.incremental.manager.IncrementalBackupStorageManager; @@ -47,19 +48,22 @@ public class ConfiguredIncrementalBackupMethod implements ConfiguredBackupMethod final int hashFactoryThreads = ThreadingUtil.getRecommendedThreadCount(); // how many threads do we use to generate the hash tree LOGGER.info("Threads: " + hashFactoryThreads); + // needed in abort progress + File levelPathFile = new File(levelPath); IncrementalBackupFeedback feedback; - try { - File levelPathFile = new File(levelPath); + IncrementalBackupStorageManager storageManager = null; + ObjectCollection2 collection = null; // this backup's collection + try { // construct incremental backup index PrintUtil.info("Hashing files..."); // TODO - ObjectCollection2 collection = new ObjectCollectionFactory<>(Sha256Identifier.getFactory(), hashFactoryThreads, 16) + collection = new ObjectCollectionFactory<>(Sha256Identifier.getFactory(), hashFactoryThreads, 16) .fromDirectory(levelPathFile, new HashSet<>(Arrays.asList("session.lock", "kbackup_metadata"))); // update storage PrintUtil.info("Copying files..."); - IncrementalBackupStorageManager storageManager = new IncrementalBackupStorageManager(Paths.get(backupBaseDirectory)); + storageManager = new IncrementalBackupStorageManager(Paths.get(backupBaseDirectory)); IncCopyResult copyResult = storageManager.addObjectCollection(collection, levelPathFile); if (copyResult == null) { PrintUtil.info("Failed to backup. No further information."); @@ -93,16 +97,32 @@ public class ConfiguredIncrementalBackupMethod implements ConfiguredBackupMethod feedback = new IncrementalBackupFeedback(false, null); } + // do clean-up if failed if (!feedback.isSuccess()) { - LOGGER.severe("Failed to backup."); - // do clean-up if failed + LOGGER.severe("Failed to backup. Cleaning up..."); + + // remove index file File backupIndexFile = new File(backupIndexFileSaveDirectory, backupIndexFileName); if (backupIndexFile.exists()) { if (!backupIndexFile.delete()) { LOGGER.warning("Failed to clean up: cannot delete file " + backupIndexFile.getName()); + return feedback; // not try to remove unused files + } + } + + // remove unused object files in the base + if (collection != null) { + try { + // collection may have been copied (partially) to the base, but we may not need them + // so we perform a clean here + // perform a clean-up + Iterable<ObjectCollection2> backups = ObjectCollectionSerializer.fromDirectory(new File(backupIndexFileSaveDirectory)); + storageManager.deleteObjectCollection(collection, levelPathFile, backups); + } catch (IOException e) { + LOGGER.warning("An exception occurred while cleaning up: " + e); } + LOGGER.info("Backup aborted."); } - //TODO: do more deep clean for object files } return feedback; |