Fix file buffer does not take effect. No idea why golang's io utility is so suck. Use ad-hoc buffered copy loop instead.

1 files changed, 50 insertions, 28 deletions

diff --git a/common/copy.go b/common/copy.go
index 6f47a62..bee6515 100644
--- a/common/copy.go
+++ b/common/copy.go
@@ -7,36 +7,58 @@ Copied from https://ixday.github.io/post/golang-cancel-copy/
 import (
 	"context"
 	"io"
+	"os"
 )
 
-// here is some syntax sugar inspired by the Tomas Senart's video,
-// it allows me to inline the Reader interface
-type readerFunc func(p []byte) (n int, err error)
-
-func (rf readerFunc) Read(p []byte) (n int, err error) { return rf(p) }
-
-// Copy slightly modified function signature:
-// - context has been added in order to propagate cancellation
-// - (undo by Keuin) I do not return the number of bytes written, has it is not useful in my use case
-func Copy(ctx context.Context, out io.Writer, in io.Reader) (written int64, err error) {
-	// Copy will call the Reader and Writer interface multiple time, in order
-	// to copy by chunk (avoiding loading the whole file in memory).
-	// I insert the ability to cancel before read time as it is the earliest
-	// possible in the call process.
-	written, err = io.Copy(out, readerFunc(func(p []byte) (int, error) {
-
-		// golang non-blocking channel: https://gobyexample.com/non-blocking-channel-operations
-		select {
-
-		// if context has been canceled
-		case <-ctx.Done():
-			// stop process and propagate "context canceled" error
-			return 0, ctx.Err()
-		default:
-			// otherwise just run default io.Reader implementation
-			return in.Read(p)
+// CopyToFileWithBuffer copies data from io.Reader to os.File with given buffer and read chunk size.
+// The reader and file won't be closed.
+// If syncFile is set, the file will be synced after every read.
+func CopyToFileWithBuffer(
+	ctx context.Context,
+	out *os.File,
+	in io.Reader,
+	buffer []byte,
+	chunkSize int,
+	syncFile bool,
+) (written int64, err error) {
+	bufSize := len(buffer)
+	off := 0   // offset to the end of data in buffer
+	nRead := 0 // how many bytes were read in the last read
+	defer func() {
+		if off+nRead > 0 {
+			// write unwritten data in buffer
+			nWrite, _ := out.Write(buffer[:off+nRead])
+			written += int64(nWrite)
+			if syncFile {
+				_ = out.Sync()
+			}
 		}
-	}))
+	}()
 
-	return
+	for {
+		if err = ctx.Err(); err != nil {
+			return
+		}
+		nRead, err = in.Read(buffer[off:Min[int](off+chunkSize, bufSize)])
+		if err != nil {
+			return
+		}
+		off += nRead
+		if off == bufSize {
+			// buffer is full
+			var nWritten int
+			nWritten, err = out.Write(buffer)
+			if err != nil {
+				return
+			}
+			if syncFile {
+				err = out.Sync()
+				if err != nil {
+					return
+				}
+			}
+			written += int64(nWritten)
+			off = 0
+		}
+	}
 }