bugfix: write error

5 files changed, 47 insertions, 50 deletions

diff --git a/build.gradle b/build.gradle
index ed90ed7..dff49ed 100644
--- a/build.gradle
+++ b/build.gradle
@@ -32,7 +32,8 @@ dependencies {
 	// PSA: Some older mods, compiled on Loom 0.2.1, might have outdated Maven POMs.
 	// You may need to force-disable transitiveness on them.
 	implementation 'junit:junit:4.13.2'
-	modImplementation 'com.clickhouse:clickhouse-http-client:0.5.0'
+	modImplementation 'com.clickhouse:clickhouse-http-client:0.4.6'
+	modImplementation 'com.google.code.gson:gson:2.10.1' // 2.10 supports deserializing java record objects
 }
 
 processResources {
diff --git a/src/main/java/com/keuin/blame/SubmitWorker.java b/src/main/java/com/keuin/blame/SubmitWorker.java
index 3c33fd6..ba4a922 100644
--- a/src/main/java/com/keuin/blame/SubmitWorker.java
+++ b/src/main/java/com/keuin/blame/SubmitWorker.java
@@ -13,9 +13,7 @@ import org.jetbrains.annotations.NotNull;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static com.clickhouse.client.ClickHouseCredentials.fromUserAndPassword;
@@ -90,27 +88,43 @@ public class SubmitWorker {
     private void bulkWrite(
             List<LogEntry> buffer,
             ClickHouseRequest.Mutation req
-    ) throws IOException, ClickHouseException {
+    ) throws IOException, ClickHouseException, ExecutionException, InterruptedException {
+        if (buffer.isEmpty()) {
+            logger.error("bulkWrite is called with empty write buffer");
+            return;
+        }
+        logger.info("bulkWrite size: " + buffer.size());
+        CompletableFuture<ClickHouseResponse> fut;
         try (var os = ClickHouseDataStreamFactory.getInstance()
-                .createPipedOutputStream(req.getConfig(), (Runnable) null);
-             var resp = req.data(os.getInputStream()).executeAndWait()) {
+                .createPipedOutputStream(req.getConfig())) {
+            fut = req.data(os.getInputStream()).execute();
             for (var el : buffer) {
                 el.write(os);
             }
-            logger.info(String.format("Written %d log entries to ClickHouse. %s",
-                    buffer.size(), resp.getSummary().toString()));
+        }
+        try (var resp = fut.get()) {
+            var summary = resp.getSummary();
+            var expected = buffer.size();
+            var actual = summary.getReadRows();
+            logger.info("Write success: " + summary.toString());
+            if (expected != actual) {
+                logger.error(String.format(
+                        "Unexpected write rows, expected %d (buffer), actual %d (write summary)",
+                        expected, actual));
+            }
         }
         buffer.clear();
     }
 
     private void run() {
         var config = DatabaseUtil.DB_CONFIG;
+        logger.info("Database: " + config.toString());
         var server = ClickHouseNode.builder()
                 .host(config.address())
                 .port(ClickHouseProtocol.HTTP, config.port())
                 // .port(ClickHouseProtocol.GRPC, Integer.getInteger("chPort", 9100))
                 // .port(ClickHouseProtocol.TCP, Integer.getInteger("chPort", 9000))
-                .database(config.table())
+                .database(config.database())
                 .credentials(fromUserAndPassword(config.username(), config.password()))
                 .build();
 
@@ -121,7 +135,12 @@ public class SubmitWorker {
             try (var client = ClickHouseClient.newInstance(server.getProtocol())) {
                 writeLoop:
                 while (true) {
-                    var result = work(client, server, batchBuffer, writeImmediately);
+                    var req = client.read(server).write()
+                            .table(config.table())
+                            .format(ClickHouseFormat.RowBinary)
+//                            .option(ClickHouseClientOption.ASYNC, false)
+                            .option(ClickHouseClientOption.COMPRESS, false);
+                    var result = work(client, req, batchBuffer, writeImmediately);
                     switch (result) {
                         case CONTINUE -> {
                             writeImmediately = false;
@@ -133,6 +152,9 @@ public class SubmitWorker {
                         case FINISH -> {
                             break workLoop;
                         }
+                        case INSTANT_WRITE -> {
+                            writeImmediately = true;
+                        }
                     }
                 }
             }
@@ -148,12 +170,10 @@ public class SubmitWorker {
 
     private @NotNull WorkResult work(
             ClickHouseClient client,
-            ClickHouseNode server,
+            ClickHouseRequest.Mutation req,
             List<LogEntry> buffer,
             boolean writeImmediately
     ) {
-        var req = client.write(server).format(ClickHouseFormat.RowBinary)
-                .option(ClickHouseClientOption.ASYNC, false);
         boolean interrupted = false;
         // if writeImmediately is set, do not accumulate the buffer, flush it immediately
         try {
@@ -177,13 +197,15 @@ public class SubmitWorker {
                 return WorkResult.FINISH;
             }
             return WorkResult.RECONNECT;
-        } catch (ClickHouseException ex) {
+        } catch (ClickHouseException | ExecutionException ex) {
             logger.error("ClickHouse writer error", ex);
             try {
                 Thread.sleep(3000);
             } catch (InterruptedException ignored) {
             }
             return WorkResult.RECONNECT;
+        } catch (InterruptedException e) {
+            return WorkResult.INSTANT_WRITE;
         }
         return WorkResult.CONTINUE;
     }
diff --git a/src/main/java/com/keuin/blame/data/entry/LogEntry.java b/src/main/java/com/keuin/blame/data/entry/LogEntry.java
index 839afdd..fba04be 100644
--- a/src/main/java/com/keuin/blame/data/entry/LogEntry.java
+++ b/src/main/java/com/keuin/blame/data/entry/LogEntry.java
@@ -10,7 +10,6 @@ import com.keuin.blame.util.UuidUtils;
 import net.minecraft.MinecraftVersion;
 
 import java.io.IOException;
-import java.io.OutputStream;
 import java.util.Objects;
 import java.util.UUID;
 
@@ -148,49 +147,34 @@ public class LogEntry {
     }
 
     public void write(ClickHousePipedOutputStream os) throws IOException {
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeInt8(os, actionType.getValue());
 
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeFixedString(os, gameVersion, 8);
 
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeString(os, objectId);
 
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeFixedString(os, objectPos.getWorld(), 24);
 
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeInt64(os, (long) (objectPos.getX()));
 
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeInt64(os, (long) (objectPos.getY()));
 
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeInt64(os, (long) (objectPos.getZ()));
 
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeInt32(os, objectType.getValue());
 
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeString(os, subjectId);
 
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeFixedString(os, subjectPos.getWorld(), 24);
 
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeFloat64(os, subjectPos.getX());
 
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeFloat64(os, subjectPos.getY());
 
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeFloat64(os, subjectPos.getZ());
 
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeFixedString(os, subjectUUID.toString(), 36); // lowercase
 
-        BinaryStreamUtils.writeNonNull(os);
         BinaryStreamUtils.writeInt64(os, timeMillis);
     }
 }
diff --git a/src/main/java/com/keuin/blame/lookup/AbstractLookupFilter.java b/src/main/java/com/keuin/blame/lookup/AbstractLookupFilter.java
deleted file mode 100644
index f2e45dc..0000000
--- a/src/main/java/com/keuin/blame/lookup/AbstractLookupFilter.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package com.keuin.blame.lookup;
-
-import org.bson.conversions.Bson;
-
-public interface AbstractLookupFilter {
-    abstract Bson filter();
-}
\ No newline at end of file
diff --git a/src/main/java/com/keuin/blame/test/TestDatabase.java b/src/main/java/com/keuin/blame/test/TestDatabase.java
index 84bf4bc..6d40eb5 100644
--- a/src/main/java/com/keuin/blame/test/TestDatabase.java
+++ b/src/main/java/com/keuin/blame/test/TestDatabase.java
@@ -6,9 +6,6 @@ import com.keuin.blame.data.WorldPos;
 import com.keuin.blame.data.entry.LogEntry;
 import com.keuin.blame.data.enums.ActionType;
 import com.keuin.blame.data.enums.ObjectType;
-import com.keuin.blame.lookup.DummyFilter;
-import com.keuin.blame.lookup.LookupCallback;
-import com.keuin.blame.lookup.LookupManager;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -70,15 +67,15 @@ public class TestDatabase {
             );
             SubmitWorker.INSTANCE.submit(entry);
             Thread.sleep(2000);
-            LookupManager.INSTANCE.lookup(new DummyFilter(), new LookupCallback() {
-                @Override
-                public void onLookupFinishes(Iterable<LogEntry> logEntries) {
-                    for (LogEntry e : logEntries) {
-                        System.out.println(e);
-                        success[0] |= Objects.equals(e, entry);
-                    }
-                }
-            }, 100);
+//            LookupManager.INSTANCE.lookup(new DummyFilter(), new LookupCallback() {
+//                @Override
+//                public void onLookupFinishes(Iterable<LogEntry> logEntries) {
+//                    for (LogEntry e : logEntries) {
+//                        System.out.println(e);
+//                        success[0] |= Objects.equals(e, entry);
+//                    }
+//                }
+//            }, 100);
             Thread.sleep(2000);
             assertTrue(success[0]);
         } catch (Exception e) {