If incremental backup failed, unfinished copy will be fully reverted.

5 files changed, 156 insertions, 52 deletions

diff --git a/src/main/java/com/keuin/kbackupfabric/KBCommands.java b/src/main/java/com/keuin/kbackupfabric/KBCommands.java
index d9213c3..9c887fd 100644
--- a/src/main/java/com/keuin/kbackupfabric/KBCommands.java
+++ b/src/main/java/com/keuin/kbackupfabric/KBCommands.java
@@ -90,6 +90,7 @@ public final class KBCommands {
         // TODO: Show real name and size and etc info for incremental backup
         // TODO: Show concrete info from metadata for `.zip` backup
         MinecraftServer server = context.getSource().getMinecraftServer();
+        // TODO: refactor this to use {@link ObjectCollectionSerializer#fromDirectory}
         File[] files = getBackupSaveDirectory(server).listFiles(
                 (dir, name) -> dir.isDirectory() &&
                         (name.toLowerCase().endsWith(".zip") && name.toLowerCase().startsWith(getBackupFileNamePrefix())
@@ -100,6 +101,7 @@ public final class KBCommands {
             Objects.requireNonNull(file);
             if (file.getName().toLowerCase().endsWith(".zip"))
                 return getPrimitiveBackupInformationString(file.getName(), file.length());
+                // TODO: refactor this to use {@link ObjectCollectionSerializer#fromDirectory}
             else if (file.getName().toLowerCase().endsWith(".kbi"))
                 return getIncrementalBackupInformationString(file);
             return file.getName();
diff --git a/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionIterator.java b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionIterator.java
new file mode 100644
index 0000000..248d36d
--- /dev/null
+++ b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionIterator.java
@@ -0,0 +1,50 @@
+package com.keuin.kbackupfabric.backup.incremental;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+public class ObjectCollectionIterator implements Iterator<ObjectElement> {
+
+    // TODO: test this
+
+    private Iterator<ObjectElement> currentIterator;
+    private final List<ObjectCollection2> cols = new LinkedList<>();
+
+    public ObjectCollectionIterator(ObjectCollection2 collection) {
+        cols.addAll(collection.getSubCollectionSet());
+        currentIterator = collection.getElementSet().iterator();
+    }
+
+    @Override
+    public boolean hasNext() {
+        if (currentIterator != null) {
+            if (currentIterator.hasNext())
+                return true;
+            else {
+                currentIterator = null;
+                return hasNext();
+            }
+        } else {
+            if (cols.isEmpty())
+                return false;
+            else {
+                ObjectCollection2 consumedCollection = cols.remove(0);
+                cols.addAll(consumedCollection.getSubCollectionSet());
+                currentIterator = consumedCollection.getElementSet().iterator();
+                return hasNext();
+            }
+        }
+    }
+
+    @Override
+    public ObjectElement next() {
+        if (hasNext()) {
+            return currentIterator.next();
+        } else {
+            throw new NoSuchElementException();
+        }
+    }
+
+}
diff --git a/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializer.java b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializer.java
index f663f20..fa411a0 100644
--- a/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializer.java
+++ b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializer.java
@@ -1,6 +1,11 @@
 package com.keuin.kbackupfabric.backup.incremental;
 
+import org.jetbrains.annotations.NotNull;
+
 import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Iterator;
 import java.util.Objects;
 
 /**
@@ -39,4 +44,40 @@ public class ObjectCollectionSerializer {
             }
         }
     }
+
+    public static Iterable<ObjectCollection2> fromDirectory(File directory) throws IOException {
+
+        if (!directory.isDirectory()) {
+            throw new IllegalArgumentException("Given directory is invalid.");
+        }
+        return new Iterable<ObjectCollection2>() {
+            private final Iterator<ObjectCollection2> iter = new Iterator<ObjectCollection2>() {
+                private final Iterator<Path> i = Files.walk(directory.toPath(), 1).filter(p -> {
+                    File f = p.toFile();
+                    return f.isFile() && f.getName().endsWith(".kbi");
+                }).iterator();
+
+                @Override
+                public boolean hasNext() {
+                    return i.hasNext();
+                }
+
+                @Override
+                public ObjectCollection2 next() {
+                    try {
+                        return fromFile(i.next().toFile());
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+            };
+
+            @NotNull
+            @Override
+            public Iterator<ObjectCollection2> iterator() {
+                return iter;
+            }
+        };
+
+    }
 }
diff --git a/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java b/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java
index be01966..ad7287f 100644
--- a/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java
+++ b/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java
@@ -1,9 +1,9 @@
 package com.keuin.kbackupfabric.backup.incremental.manager;
 
 import com.keuin.kbackupfabric.backup.incremental.ObjectCollection2;
+import com.keuin.kbackupfabric.backup.incremental.ObjectCollectionIterator;
 import com.keuin.kbackupfabric.backup.incremental.ObjectElement;
 import com.keuin.kbackupfabric.backup.incremental.identifier.ObjectIdentifier;
-import com.keuin.kbackupfabric.backup.incremental.identifier.StorageObjectLoader;
 import com.keuin.kbackupfabric.util.FilesystemUtil;
 import com.keuin.kbackupfabric.util.PrintUtil;
 import org.jetbrains.annotations.Nullable;
@@ -14,6 +14,7 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
+import java.util.logging.Logger;
 
 import static org.apache.commons.io.FileUtils.forceDelete;
 
@@ -23,6 +24,8 @@ public class IncrementalBackupStorageManager {
     private final Map<ObjectIdentifier, File> map = new HashMap<>();
     private boolean loaded = false;
 
+    private final Logger LOGGER = Logger.getLogger(IncrementalBackupStorageManager.class.getName());
+
     public IncrementalBackupStorageManager(Path backupStorageBase) {
         this.backupStorageBase = backupStorageBase;
     }
@@ -32,7 +35,7 @@ public class IncrementalBackupStorageManager {
      *
      * @param collection the collection.
      * @return objects copied to the base.
-     * @throws IOException I/O Error.
+     * @throws IOException I/O error.
      */
     public @Nullable
     IncCopyResult addObjectCollection(ObjectCollection2 collection, File collectionBasePath) throws IOException {
@@ -71,6 +74,37 @@ public class IncrementalBackupStorageManager {
     }
 
     /**
+     * Delete all files in the specific collection, from the storage base.
+     *
+     * @param collection         the collection containing files to be deleted.
+     * @param collectionBasePath the collection base path.
+     * @throws IOException I/O error.
+     */
+    public void deleteObjectCollection(ObjectCollection2 collection, File collectionBasePath) throws IOException {
+        deleteObjectCollection(collection, collectionBasePath, Collections.emptySet());
+    }
+
+    /**
+     * Delete a collection from the storage base, optionally preserving files used by other backups.
+     *
+     * @param collection               the collection containing files to be deleted.
+     * @param collectionBasePath       the collection base path.
+     * @param otherExistingCollections other collections (not to be deleted) in this base. Files exist in these collections will not be deleted.
+     */
+    public void deleteObjectCollection(ObjectCollection2 collection, File collectionBasePath,
+                                       Iterable<ObjectCollection2> otherExistingCollections) {
+        Iterator<ObjectElement> iter = new ObjectCollectionIterator(collection);
+        Set<ObjectElement> unusedElementSet = new HashSet<>();
+        iter.forEachRemaining(unusedElementSet::add);
+        otherExistingCollections.forEach(col -> new ObjectCollectionIterator(col).forEachRemaining(unusedElementSet::remove));
+        unusedElementSet.forEach(ele -> {
+            File file = new File(backupStorageBase.toFile(), ele.getIdentifier().getIdentification());
+            if (!file.delete())
+                LOGGER.warning("Failed to delete unused file " + file.getName());
+        });
+    }
+
+    /**
      * Restore an object collection from the storage base. i.e., restore the save from backup storage.
      *
      * @param collection         the collection to be restored.
@@ -140,28 +174,6 @@ public class IncrementalBackupStorageManager {
         return copyCount;
     }
 
-    public int cleanUnusedObjects(Iterable<ObjectCollection2> collectionIterable) {
-        // construct object list in memory
-        Set<String> objects = new HashSet<>();
-//        backupStorageBase
-
-        for (ObjectCollection2 collection : collectionIterable) {
-            for (ObjectElement ele : collection.getElementMap().values()) {
-
-            }
-        }
-        throw new RuntimeException("not impl");
-    }
-
-    /**
-     * Check all objects, return unused ones.
-     *
-     * @return the unused ones.
-     */
-    private Map<ObjectIdentifier, File> markUnusedObjects() {
-        throw new RuntimeException("not impl");
-    }
-
     /**
      * Check if the backup base contains given element.
      *
@@ -174,25 +186,4 @@ public class IncrementalBackupStorageManager {
         return (new File(backupStorageBase.toFile(), objectElement.getIdentifier().getIdentification())).exists();
     }
 
-    private void lazyLoadStorage() throws IOException {
-        if (!loaded) {
-            loadStorage();
-            loaded = true;
-        }
-    }
-
-    private synchronized void loadStorage() throws IOException {
-        map.clear();
-        Files.walk(backupStorageBase, 1).forEach(path -> {
-            File file = path.toFile();
-            ObjectIdentifier identifier = StorageObjectLoader.asIdentifier(file);
-            if (identifier == null) {
-                map.clear();
-                throw new IllegalStateException(String.format(
-                        "Bad storage object %s: cannot recognize identifier.", file.getName()
-                ));
-            }
-            map.put(identifier, file);
-        });
-    }
 }
diff --git a/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java b/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java
index 2a9cbc8..ffcc000 100644
--- a/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java
+++ b/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java
@@ -2,6 +2,7 @@ package com.keuin.kbackupfabric.operation.backup.method;
 
 import com.keuin.kbackupfabric.backup.incremental.ObjectCollection2;
 import com.keuin.kbackupfabric.backup.incremental.ObjectCollectionFactory;
+import com.keuin.kbackupfabric.backup.incremental.ObjectCollectionSerializer;
 import com.keuin.kbackupfabric.backup.incremental.identifier.Sha256Identifier;
 import com.keuin.kbackupfabric.backup.incremental.manager.IncCopyResult;
 import com.keuin.kbackupfabric.backup.incremental.manager.IncrementalBackupStorageManager;
@@ -47,19 +48,22 @@ public class ConfiguredIncrementalBackupMethod implements ConfiguredBackupMethod
         final int hashFactoryThreads = ThreadingUtil.getRecommendedThreadCount(); // how many threads do we use to generate the hash tree
         LOGGER.info("Threads: " + hashFactoryThreads);
 
+        // needed in abort progress
+        File levelPathFile = new File(levelPath);
         IncrementalBackupFeedback feedback;
-        try {
-            File levelPathFile = new File(levelPath);
+        IncrementalBackupStorageManager storageManager = null;
 
+        ObjectCollection2 collection = null; // this backup's collection
+        try {
             // construct incremental backup index
             PrintUtil.info("Hashing files...");
             // TODO
-            ObjectCollection2 collection = new ObjectCollectionFactory<>(Sha256Identifier.getFactory(), hashFactoryThreads, 16)
+            collection = new ObjectCollectionFactory<>(Sha256Identifier.getFactory(), hashFactoryThreads, 16)
                     .fromDirectory(levelPathFile, new HashSet<>(Arrays.asList("session.lock", "kbackup_metadata")));
 
             // update storage
             PrintUtil.info("Copying files...");
-            IncrementalBackupStorageManager storageManager = new IncrementalBackupStorageManager(Paths.get(backupBaseDirectory));
+            storageManager = new IncrementalBackupStorageManager(Paths.get(backupBaseDirectory));
             IncCopyResult copyResult = storageManager.addObjectCollection(collection, levelPathFile);
             if (copyResult == null) {
                 PrintUtil.info("Failed to backup. No further information.");
@@ -93,16 +97,32 @@ public class ConfiguredIncrementalBackupMethod implements ConfiguredBackupMethod
             feedback = new IncrementalBackupFeedback(false, null);
         }
 
+        // do clean-up if failed
         if (!feedback.isSuccess()) {
-            LOGGER.severe("Failed to backup.");
-            // do clean-up if failed
+            LOGGER.severe("Failed to backup. Cleaning up...");
+
+            // remove index file
             File backupIndexFile = new File(backupIndexFileSaveDirectory, backupIndexFileName);
             if (backupIndexFile.exists()) {
                 if (!backupIndexFile.delete()) {
                     LOGGER.warning("Failed to clean up: cannot delete file " + backupIndexFile.getName());
+                    return feedback; // not try to remove unused files
+                }
+            }
+
+            // remove unused object files in the base
+            if (collection != null) {
+                try {
+                    // collection may have been copied (partially) to the base, but we may not need them
+                    // so we perform a clean here
+                    // perform a clean-up
+                    Iterable<ObjectCollection2> backups = ObjectCollectionSerializer.fromDirectory(new File(backupIndexFileSaveDirectory));
+                    storageManager.deleteObjectCollection(collection, levelPathFile, backups);
+                } catch (IOException e) {
+                    LOGGER.warning("An exception occurred while cleaning up: " + e);
                 }
+                LOGGER.info("Backup aborted.");
             }
-            //TODO: do more deep clean for object files
         }
 
         return feedback;