From 7e3cd92742383c43f2741449c551208e6487154e Mon Sep 17 00:00:00 2001 From: Keuin Date: Sat, 20 Jan 2024 13:20:41 +0800 Subject: feature: configurable CoW incremental backup --- README.md | 85 ++++++++++++++++------ build.gradle | 21 +++++- gradle.properties | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- .../keuin/kbackupfabric/KBCommandsRegister.java | 4 + .../com/keuin/kbackupfabric/KBPluginEvents.java | 11 ++- .../manager/IncrementalBackupStorageManager.java | 26 ++++++- .../keuin/kbackupfabric/config/KBackupConfig.java | 56 ++++++++++++++ .../method/ConfiguredIncrementalBackupMethod.java | 1 + .../com/keuin/kbackupfabric/ui/KBCommands.java | 11 +++ .../keuin/kbackupfabric/util/cow/FileCopier.java | 9 +++ .../kbackupfabric/util/cow/FileCowCopier.java | 44 +++++++++++ .../kbackupfabric/util/cow/FileEagerCopier.java | 17 +++++ 13 files changed, 261 insertions(+), 28 deletions(-) create mode 100644 src/main/java/com/keuin/kbackupfabric/config/KBackupConfig.java create mode 100644 src/main/java/com/keuin/kbackupfabric/util/cow/FileCopier.java create mode 100644 src/main/java/com/keuin/kbackupfabric/util/cow/FileCowCopier.java create mode 100644 src/main/java/com/keuin/kbackupfabric/util/cow/FileEagerCopier.java diff --git a/README.md b/README.md index f39fc12..e3a95e5 100644 --- a/README.md +++ b/README.md @@ -57,26 +57,44 @@ Supported Minecraft version: 1.14.4, 1.15.2, 1.16.4/1.16.5, 1.17.1, 1.18.1 - **/kb delete**: delete an existing backup. - **/kb prev**: Find and select the most recent backup file. After executing this command, you can use `/kb restore 1` to restore to this backup. +- **/kb cow-info**: Check if copy-on-write backup is activated. Only OPs can make backups and restore by default. However, you can use permission management mods like [LuckPerms](https://luckperms.net/) to configure exactly what permissions normal players can use. Permission nodes of each command are listed below: -| Command | Permission Required | -|------------|---------------------| -| /kb | kb.root | -| /kb help | kb.help | -| /kb list | kb.list | -| /kb backup | kb.backup | -| /kb incbak | kb.incbak | +| Command | Permission Required | +|-------------|---------------------| +| /kb | kb.root | +| /kb help | kb.help | +| /kb list | kb.list | +| /kb backup | kb.backup | +| /kb incbak | kb.incbak | | /kb restore | kb.restore | -| /kb delete | kb.delete | +| /kb delete | kb.delete | | /kb confirm | kb.confirm | -| /kb cancel | kb.cancel | -| /kb prev | kb.prev | +| /kb cancel | kb.cancel | +| /kb prev | kb.prev | -## 2.2 Script for auto-restart after restoring +## 2.2 Configurations + +Since version `1.8.0`, plugin-wide config file `kbackup.json` is added in the server root directory (workdir). + +The config file will be generated if not exist. The default is: + +```json +{ + "incbak_cow": false +} +``` + +- `incbak_cow`: (experimental) Enable filesystem CoW (copy-on-write) for incremental backup. + **Will fall back to normal file copy if the filesystem does not support CoW.** Please read section 2.5 for more info. + +Note: JSON comment is **NOT** supported for now. + +## 2.3 Script for auto-restart after restoring Due to the nature of JVM: the Java language's running environment, there is no elegant way to restart Minecraft server in a server plugin. In order to auto restart after restoring, an outer system-based script is required, i.e. a batch or @@ -88,7 +106,7 @@ exit code and restart Minecraft server if the code is `111`. I will give examples for some popular operating systems. To use these scripts, you should replace your start.bat or start.sh script with given code lines. -### 2.2.1 Script for Windows (Batch script) +### 2.3.1 Script for Windows (Batch script) ```batch @echo off @@ -100,7 +118,7 @@ rem kbackup restore auto restart pause ``` -### 2.2.2 Script for Linux (Shell script) +### 2.3.2 Script for Linux (Shell script) ```shell #!/bin/sh @@ -112,13 +130,17 @@ do done ``` -## 2.3 Automatic Regular Backup +## 2.4 Automatic Regular Backup -Currently KBackup does not support automatic backup by itself. However, If application level scheduled tasks are available to you, such as *crontab* in Linux and *Task Scheduler* in Windows, you can use that to trigger backup tasks regularly. +Currently, KBackup does not support automatic backup by itself. However, If application level scheduled tasks are +available to you, such as *crontab* in Linux and *Task Scheduler* in Windows, you can use that to trigger backup tasks +regularly. -### 2.3.1 On Linux +### 2.4.1 On Linux -In order to run Minecraft command on your server as a Shell command, you need RCON client like [mcrcon](https://github.com/Tiiffi/mcrcon). You can get the binary executable from its homepage and put it into anywhere like `/usr/bin`. +In order to run Minecraft command on your server as a Shell command, you need RCON client +like [mcrcon](https://github.com/Tiiffi/mcrcon). You can get the binary executable from its homepage and put it into +anywhere like `/usr/bin`. Let's assume you are under Linux, run `crontab -e` and append this line to the configuration: @@ -128,13 +150,34 @@ Let's assume you are under Linux, run `crontab -e` and append this line to the c You can specify RCON port and password in `server.properties`. -This will cause `cron` to run `kb backup` for every 6 hours. To make incremental backups, simply replace `kb backup` to `kb incbak`. +This will cause `cron` to run `kb backup` for every 6 hours. To make incremental backups, simply replace `kb backup` +to `kb incbak`. + +The man page [crontab(5)](https://man7.org/linux/man-pages/man5/crontab.5.html) also contains useful information +about using cron. + +### 2.4.2 On Windows + +For Windows users, please refer +to [tutorials available on Google](https://www.google.com/search?q=create+scheduled+task+in+windows) for creating +scheduled tasks. Note that mcrcon is also available on Windows. + +## 2.5 Copy-on-write (CoW) backup + +CoW will save even more disk space when making backups. + +This feature requires platform-specific library [KBackup-cow](https://github.com/keuin/KBackup-cow). +Currently only Linux on x86_64 platform is supported. +Please create a GitHub issue if you want other platforms. -The man page [crontab(5)](https://man7.org/linux/man-pages/man5/crontab.5.html) also contains many useful information about using cron. +This is an experimental feature. Please report if you encountered any error. -### 2.3.2 On Windows +Requirements: -For Windows users, please refer to [tutorials available on Google](https://www.google.com/search?q=create+scheduled+task+in+windows) for creating scheduled tasks. Note that mcrcon is also available on Windows. +1. The server is running Linux on amd64. +2. Filesystem should support CoW (e.g. Bcachefs, Btrfs, XFS, OCFS2, ReFSv2). Please + read for more info about CoW filesystems. +3. Backup directory and server world directory should on the same disk partition. This is required to clone a file. # 3. To-Do List diff --git a/build.gradle b/build.gradle index 67b3e86..ce89a4e 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,7 @@ plugins { id 'fabric-loom' version '1.0-SNAPSHOT' id 'maven-publish' + id 'com.github.johnrengelman.shadow' version '7.1.2' } sourceCompatibility = JavaVersion.VERSION_1_8 @@ -20,6 +21,20 @@ repositories { // for more information about repositories. } +shadowJar { + configurations = [project.configurations.shadow] + + // mitigate log4j security problem + exclude 'org/apache/logging/log4j/core/lookup/JndiLookup.class' + exclude 'META-INF' +} + +remapJar { + dependsOn(shadowJar) + mustRunAfter(shadowJar) + inputFile = tasks.shadowJar.archiveFile +} + dependencies { implementation 'junit:junit:4.13.2' @@ -31,6 +46,10 @@ dependencies { // Fabric API. This is technically optional, but you probably want it anyway. modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}" + // https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind + modImplementation 'com.fasterxml.jackson.core:jackson-databind:2.13.5' + shadow 'com.fasterxml.jackson.core:jackson-databind:2.13.5' + // Uncomment the following line to enable the deprecated Fabric API modules. // These are included in the Fabric API production distribution and allow you to update your mod to the latest modules at a later more convenient time. @@ -54,7 +73,7 @@ java { jar { from("LICENSE") { - rename { "${it}_${project.archivesBaseName}"} + rename { "${it}_${project.archivesBaseName}" } } } diff --git a/gradle.properties b/gradle.properties index 81cb9b8..f9215ba 100644 --- a/gradle.properties +++ b/gradle.properties @@ -6,7 +6,7 @@ minecraft_version=1.14.4 yarn_mappings=1.14.4+build.18 loader_version=0.14.12 # Mod Properties -mod_version=1.7.0 +mod_version=1.8.0 maven_group=com.keuin.kbackupfabric archives_base_name=kbackup-fabric # Dependencies diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ffed3a2..db9a6b8 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/com/keuin/kbackupfabric/KBCommandsRegister.java b/src/main/java/com/keuin/kbackupfabric/KBCommandsRegister.java index 5bf92e4..7e454d2 100644 --- a/src/main/java/com/keuin/kbackupfabric/KBCommandsRegister.java +++ b/src/main/java/com/keuin/kbackupfabric/KBCommandsRegister.java @@ -58,6 +58,10 @@ public final class KBCommandsRegister { .requires(Permissions.require("kb.list", DEFAULT_REQUIRED_LEVEL)) .executes(KBCommands::list))); + dispatcher.register(CommandManager.literal("kb") + .then(CommandManager.literal("cow-info") + .executes(KBCommands::cowInfo))); + // register /kb delete [name] for deleting an existing backup. OP is required. dispatcher.register(CommandManager.literal("kb") .then(CommandManager.literal("delete") diff --git a/src/main/java/com/keuin/kbackupfabric/KBPluginEvents.java b/src/main/java/com/keuin/kbackupfabric/KBPluginEvents.java index 362abb8..20e9058 100644 --- a/src/main/java/com/keuin/kbackupfabric/KBPluginEvents.java +++ b/src/main/java/com/keuin/kbackupfabric/KBPluginEvents.java @@ -2,6 +2,7 @@ package com.keuin.kbackupfabric; import com.keuin.kbackupfabric.backup.BackupFilesystemUtil; import com.keuin.kbackupfabric.backup.suggestion.BackupNameSuggestionProvider; +import com.keuin.kbackupfabric.config.KBackupConfig; import com.keuin.kbackupfabric.event.OnPlayerConnect; import com.keuin.kbackupfabric.metadata.BackupMetadata; import com.keuin.kbackupfabric.metadata.MetadataHolder; @@ -32,7 +33,15 @@ public final class KBPluginEvents implements ModInitializer { @Override public void onInitialize() { - System.out.println("Binding events and commands ..."); + PrintUtil.info("Reading config..."); + try { + KBackupConfig.load(); + } catch (IOException e) { + PrintUtil.error("Failed to read config file, using default values: " + e + e.getMessage()); + } + boolean incCow = KBackupConfig.getInstance().getIncbakCow(); + PrintUtil.info("Incremental backup CoW: " + (incCow ? "enabled" : "disabled")); + PrintUtil.info("Binding events and commands..."); CommandRegistrationCallback.EVENT.register(KBCommandsRegister::registerCommands); OnPlayerConnect.ON_PLAYER_CONNECT.register((connection, player) -> NotificationManager.INSTANCE.notifyPlayer(DistinctNotifiable.fromServerPlayerEntity(player))); diff --git a/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java b/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java index 7870620..53143f8 100644 --- a/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java +++ b/src/main/java/com/keuin/kbackupfabric/backup/incremental/manager/IncrementalBackupStorageManager.java @@ -4,15 +4,18 @@ import com.keuin.kbackupfabric.backup.incremental.ObjectCollection2; import com.keuin.kbackupfabric.backup.incremental.ObjectCollectionIterator; import com.keuin.kbackupfabric.backup.incremental.ObjectElement; import com.keuin.kbackupfabric.backup.incremental.identifier.ObjectIdentifier; +import com.keuin.kbackupfabric.config.KBackupConfig; import com.keuin.kbackupfabric.util.FilesystemUtil; import com.keuin.kbackupfabric.util.PrintUtil; +import com.keuin.kbackupfabric.util.cow.FileCopier; +import com.keuin.kbackupfabric.util.cow.FileCowCopier; +import com.keuin.kbackupfabric.util.cow.FileEagerCopier; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import java.util.logging.Logger; @@ -27,9 +30,26 @@ public class IncrementalBackupStorageManager { private final Logger logger = Logger.getLogger(IncrementalBackupStorageManager.class.getName()); private final Path backupStorageBase; private final Logger LOGGER = Logger.getLogger(IncrementalBackupStorageManager.class.getName()); + private FileCopier copier; public IncrementalBackupStorageManager(Path backupStorageBase) { this.backupStorageBase = backupStorageBase; + if (KBackupConfig.getInstance().getIncbakCow()) { + // try to use cow copier, if failed, fallback to normal copier + try { + this.copier = FileCowCopier.getInstance(); + } catch (Exception | UnsatisfiedLinkError ex) { + PrintUtil.error("Failed to initialize kbackup-cow: " + ex + ex.getMessage()); + this.copier = new FileEagerCopier(); + } + } else { + this.copier = new FileEagerCopier(); + } + if (this.copier.isCow()) { + PrintUtil.info("Copy-on-write is enabled"); + } else { + PrintUtil.info("Copy-on-write is disabled"); + } } /** @@ -85,7 +105,7 @@ public class IncrementalBackupStorageManager { if (!contains(entry.getValue())) { // element does not exist. copy. logger.fine("Copy new file `" + copySourceFile.getName() + "`."); - Files.copy(copySourceFile.toPath(), copyDestination.toPath()); + copier.copy(copyDestination.getAbsolutePath(), copySourceFile.getAbsolutePath()); copyCount = copyCount.addWith(new IncCopyResult(1, 1, fileBytes, fileBytes)); } else { // element exists (file reused). Just update the stat info @@ -198,7 +218,7 @@ public class IncrementalBackupStorageManager { } } - Files.copy(copySource.toPath(), copyTarget.toPath()); + copier.copy(copyTarget.getAbsolutePath(), copySource.getAbsolutePath()); ++copyCount; } diff --git a/src/main/java/com/keuin/kbackupfabric/config/KBackupConfig.java b/src/main/java/com/keuin/kbackupfabric/config/KBackupConfig.java new file mode 100644 index 0000000..8ece8d2 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/config/KBackupConfig.java @@ -0,0 +1,56 @@ +package com.keuin.kbackupfabric.config; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.keuin.kbackupfabric.util.PrintUtil; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; + +public class KBackupConfig { + + private static KBackupConfig instance = getDefault(); + private static final String CONFIG_FILE = "kbackup.json"; + + @JsonProperty("incbak_cow") + private Boolean incbakCow; + + public static KBackupConfig getInstance() { + return instance; + } + + private static KBackupConfig getDefault() { + return new KBackupConfig(false); + } + + public static void load() throws IOException { + File file = new File(CONFIG_FILE); + ObjectMapper om = new ObjectMapper(); + try { + instance = om.readValue(file, KBackupConfig.class); + } catch (FileNotFoundException ignored) { + // generate default config file + PrintUtil.info("Config file does not exist. Creating default config: " + CONFIG_FILE); + instance = getDefault(); + ObjectWriter w = om.writerWithDefaultPrettyPrinter(); + w.writeValue(file, instance); + } + } + + public KBackupConfig() { + } + + public KBackupConfig(Boolean incbakCow) { + this.incbakCow = incbakCow; + } + + public Boolean getIncbakCow() { + return this.incbakCow; + } + + public void setIncbakCow(Boolean incbakCow) { + this.incbakCow = incbakCow; + } +} \ No newline at end of file diff --git a/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java b/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java index cfc9b81..3d9007d 100644 --- a/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java +++ b/src/main/java/com/keuin/kbackupfabric/operation/backup/method/ConfiguredIncrementalBackupMethod.java @@ -99,6 +99,7 @@ public class ConfiguredIncrementalBackupMethod implements ConfiguredBackupMethod PrintUtil.info("Incremental backup finished."); feedback = new IncrementalBackupFeedback(true, copyResult); } catch (IOException e) { + PrintUtil.error("Incremental backup failed: " + e + e.getMessage()); feedback = new IncrementalBackupFeedback(e); } diff --git a/src/main/java/com/keuin/kbackupfabric/ui/KBCommands.java b/src/main/java/com/keuin/kbackupfabric/ui/KBCommands.java index d23a6fd..0dff99a 100644 --- a/src/main/java/com/keuin/kbackupfabric/ui/KBCommands.java +++ b/src/main/java/com/keuin/kbackupfabric/ui/KBCommands.java @@ -14,6 +14,7 @@ import com.keuin.kbackupfabric.operation.backup.method.ConfiguredIncrementalBack import com.keuin.kbackupfabric.operation.backup.method.ConfiguredPrimitiveBackupMethod; import com.keuin.kbackupfabric.util.DateUtil; import com.keuin.kbackupfabric.util.PrintUtil; +import com.keuin.kbackupfabric.util.cow.FileCowCopier; import com.mojang.brigadier.arguments.StringArgumentType; import com.mojang.brigadier.context.CommandContext; import net.minecraft.server.MinecraftServer; @@ -75,6 +76,7 @@ public final class KBCommands { msgInfo(context, "/kb restore - Delete the whole current level and restore from given backup. /kb restore is identical with /kb list."); msgInfo(context, "/kb confirm - Confirm and start restoring."); msgInfo(context, "/kb cancel - Cancel the restoration to be confirmed. If cancelled, /kb confirm will not run."); + msgInfo(context, "/kb cow-info - Display copy-on-write support info (Experimental)"); return SUCCESS; } @@ -135,6 +137,15 @@ public final class KBCommands { return SUCCESS; } + public static int cowInfo(CommandContext context) { + try { + msgInfo(context, "KBackup-cow library version: " + FileCowCopier.getVersion()); + } catch (Exception | UnsatisfiedLinkError ignored) { + msgErr(context, "KBackup-cow library is not loaded"); + } + return SUCCESS; + } + /** * Print backup information. * diff --git a/src/main/java/com/keuin/kbackupfabric/util/cow/FileCopier.java b/src/main/java/com/keuin/kbackupfabric/util/cow/FileCopier.java new file mode 100644 index 0000000..eb21af7 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/cow/FileCopier.java @@ -0,0 +1,9 @@ +package com.keuin.kbackupfabric.util.cow; + +import java.io.IOException; + +public interface FileCopier { + void copy(String dst, String src) throws IOException; + + boolean isCow(); +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/cow/FileCowCopier.java b/src/main/java/com/keuin/kbackupfabric/util/cow/FileCowCopier.java new file mode 100644 index 0000000..e5819de --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/cow/FileCowCopier.java @@ -0,0 +1,44 @@ +package com.keuin.kbackupfabric.util.cow; + +import com.keuin.kbackupfabric.util.PrintUtil; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; + +public final class FileCowCopier { + private static final AtomicBoolean initialized = new AtomicBoolean(false); + + static { + try { + System.loadLibrary("kbackup_cow"); + } catch (SecurityException | UnsatisfiedLinkError ignored) { + } + } + + public static native void init(); + + public static native void copy(String dst, String src) throws IOException; + + public static native String getVersion(); + + public static FileCopier getInstance() { + if (initialized.compareAndSet(false, true)) { + FileCowCopier.init(); + PrintUtil.info("kbackup-cow version: " + FileCowCopier.getVersion()); + } + // call a native method to ensure the dynamic library is correctly loaded, JVM will throw if failed + // so the outside fallback logic could work + FileCowCopier.getVersion(); + return new FileCopier() { + @Override + public void copy(String dst, String src) throws IOException { + FileCowCopier.copy(dst, src); + } + + @Override + public boolean isCow() { + return true; + } + }; + } +} diff --git a/src/main/java/com/keuin/kbackupfabric/util/cow/FileEagerCopier.java b/src/main/java/com/keuin/kbackupfabric/util/cow/FileEagerCopier.java new file mode 100644 index 0000000..0f12374 --- /dev/null +++ b/src/main/java/com/keuin/kbackupfabric/util/cow/FileEagerCopier.java @@ -0,0 +1,17 @@ +package com.keuin.kbackupfabric.util.cow; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + +public class FileEagerCopier implements FileCopier { + @Override + public void copy(String dst, String src) throws IOException { + Files.copy(Paths.get(src), Paths.get(dst)); + } + + @Override + public boolean isCow() { + return false; + } +} -- cgit v1.2.3