From 1df50093bd76315905a9aae880470e81b5e1d8f0 Mon Sep 17 00:00:00 2001 From: Keuin Date: Sun, 24 Jan 2021 21:15:24 +0800 Subject: If incremental backup failed, unfinished copy will be fully reverted. --- .../incremental/ObjectCollectionIterator.java | 50 +++++++++++++ .../incremental/ObjectCollectionSerializer.java | 41 +++++++++++ .../manager/IncrementalBackupStorageManager.java | 81 ++++++++++------------ 3 files changed, 127 insertions(+), 45 deletions(-) create mode 100644 src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionIterator.java (limited to 'src/main/java/com/keuin/kbackupfabric/backup/incremental') diff --git a/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionIterator.java b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionIterator.java new file mode 100644 index 0000000..248d36d --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionIterator.java @@ -0,0 +1,50 @@ +package com.keuin.kbackupfabric.backup.incremental; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.NoSuchElementException; + +public class ObjectCollectionIterator implements Iterator { + + // TODO: test this + + private Iterator currentIterator; + private final List cols = new LinkedList<>(); + + public ObjectCollectionIterator(ObjectCollection2 collection) { + cols.addAll(collection.getSubCollectionSet()); + currentIterator = collection.getElementSet().iterator(); + } + + @Override + public boolean hasNext() { + if (currentIterator != null) { + if (currentIterator.hasNext()) + return true; + else { + currentIterator = null; + return hasNext(); + } + } else { + if (cols.isEmpty()) + return false; + else { + ObjectCollection2 consumedCollection = cols.remove(0); + cols.addAll(consumedCollection.getSubCollectionSet()); + currentIterator = consumedCollection.getElementSet().iterator(); + return hasNext(); + } + } + } + + @Override + public ObjectElement next() { + if (hasNext()) { + return currentIterator.next(); + } else { + throw new NoSuchElementException(); + } + } + +} diff --git a/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializer.java b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializer.java index f663f20..fa411a0 100644 --- a/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializer.java +++ b/src/main/java/com/keuin/kbackupfabric/backup/incremental/ObjectCollectionSerializer.java @@ -1,6 +1,11 @@ package com.keuin.kbackupfabric.backup.incremental; +import org.jetbrains.annotations.NotNull; + import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; import java.util.Objects; /** @@ -39,4 +44,40 @@ public class ObjectCollectionSerializer { } } } + + public static Iterable fromDirectory(File directory) throws IOException { + + if (!directory.isDirectory()) { + throw new IllegalArgumentException("Given directory is invalid."); + } + return new Iterable() { + private final Iterator iter = new Iterator() { + private final Iterator i = Files.walk(directory.toPath(), 1).filter(p -> { + File f = p.toFile(); + return f.isFile() && f.getName().endsWith(".kbi"); + }).iterator(); + + @Override + public boolean hasNext() { + return i.hasNext(); + } + + @Override + public ObjectCollection2 next() { + try { + return fromFile(i.next().toFile()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }; + + @NotNull + @Override + public Iterator iterator() { + return iter; + } + }; + + } } diff --git a/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java b/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java index be01966..ad7287f 100644 --- a/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java +++ b/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java @@ -1,9 +1,9 @@ package com.keuin.kbackupfabric.backup.incremental.manager; import com.keuin.kbackupfabric.backup.incremental.ObjectCollection2; +import com.keuin.kbackupfabric.backup.incremental.ObjectCollectionIterator; import com.keuin.kbackupfabric.backup.incremental.ObjectElement; import com.keuin.kbackupfabric.backup.incremental.identifier.ObjectIdentifier; -import com.keuin.kbackupfabric.backup.incremental.identifier.StorageObjectLoader; import com.keuin.kbackupfabric.util.FilesystemUtil; import com.keuin.kbackupfabric.util.PrintUtil; import org.jetbrains.annotations.Nullable; @@ -14,6 +14,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; +import java.util.logging.Logger; import static org.apache.commons.io.FileUtils.forceDelete; @@ -23,6 +24,8 @@ public class IncrementalBackupStorageManager { private final Map map = new HashMap<>(); private boolean loaded = false; + private final Logger LOGGER = Logger.getLogger(IncrementalBackupStorageManager.class.getName()); + public IncrementalBackupStorageManager(Path backupStorageBase) { this.backupStorageBase = backupStorageBase; } @@ -32,7 +35,7 @@ public class IncrementalBackupStorageManager { * * @param collection the collection. * @return objects copied to the base. - * @throws IOException I/O Error. + * @throws IOException I/O error. */ public @Nullable IncCopyResult addObjectCollection(ObjectCollection2 collection, File collectionBasePath) throws IOException { @@ -70,6 +73,37 @@ public class IncrementalBackupStorageManager { return copyCount; } + /** + * Delete all files in the specific collection, from the storage base. + * + * @param collection the collection containing files to be deleted. + * @param collectionBasePath the collection base path. + * @throws IOException I/O error. + */ + public void deleteObjectCollection(ObjectCollection2 collection, File collectionBasePath) throws IOException { + deleteObjectCollection(collection, collectionBasePath, Collections.emptySet()); + } + + /** + * Delete a collection from the storage base, optionally preserving files used by other backups. + * + * @param collection the collection containing files to be deleted. + * @param collectionBasePath the collection base path. + * @param otherExistingCollections other collections (not to be deleted) in this base. Files exist in these collections will not be deleted. + */ + public void deleteObjectCollection(ObjectCollection2 collection, File collectionBasePath, + Iterable otherExistingCollections) { + Iterator iter = new ObjectCollectionIterator(collection); + Set unusedElementSet = new HashSet<>(); + iter.forEachRemaining(unusedElementSet::add); + otherExistingCollections.forEach(col -> new ObjectCollectionIterator(col).forEachRemaining(unusedElementSet::remove)); + unusedElementSet.forEach(ele -> { + File file = new File(backupStorageBase.toFile(), ele.getIdentifier().getIdentification()); + if (!file.delete()) + LOGGER.warning("Failed to delete unused file " + file.getName()); + }); + } + /** * Restore an object collection from the storage base. i.e., restore the save from backup storage. * @@ -140,28 +174,6 @@ public class IncrementalBackupStorageManager { return copyCount; } - public int cleanUnusedObjects(Iterable collectionIterable) { - // construct object list in memory - Set objects = new HashSet<>(); -// backupStorageBase - - for (ObjectCollection2 collection : collectionIterable) { - for (ObjectElement ele : collection.getElementMap().values()) { - - } - } - throw new RuntimeException("not impl"); - } - - /** - * Check all objects, return unused ones. - * - * @return the unused ones. - */ - private Map markUnusedObjects() { - throw new RuntimeException("not impl"); - } - /** * Check if the backup base contains given element. * @@ -174,25 +186,4 @@ public class IncrementalBackupStorageManager { return (new File(backupStorageBase.toFile(), objectElement.getIdentifier().getIdentification())).exists(); } - private void lazyLoadStorage() throws IOException { - if (!loaded) { - loadStorage(); - loaded = true; - } - } - - private synchronized void loadStorage() throws IOException { - map.clear(); - Files.walk(backupStorageBase, 1).forEach(path -> { - File file = path.toFile(); - ObjectIdentifier identifier = StorageObjectLoader.asIdentifier(file); - if (identifier == null) { - map.clear(); - throw new IllegalStateException(String.format( - "Bad storage object %s: cannot recognize identifier.", file.getName() - )); - } - map.put(identifier, file); - }); - } } -- cgit v1.2.3