Refactor and Optimize WAL implementation

.gitignore

@@ -37,3 +37,4 @@ pnpm-debug.log*
 .DS_Store
 dicedb.yaml
 dicedb.conf
+logs/

Makefile

@@ -38,7 +38,7 @@ unittest-one: ## run a single unit test function by name (e.g. make unittest-one
 
 ##@ Development
 run: ## run dicedb with the default configuration
-	go run main.go --engine ironhawk --log-level debug
+	go run main.go --engine ironhawk --log-level debug --enable-wal
 
 format: ## format the code using go fmt
 	go fmt ./...

config/config.go

@@ -62,7 +62,7 @@ type DiceDBConfig struct {
 	Engine string `mapstructure:"engine" default:"ironhawk" description:"the engine to use, values: ironhawk"`
 
 	EnableWAL                         bool   `mapstructure:"enable-wal" default:"false" description:"enable write-ahead logging"`
-	WALDir                            string `mapstructure:"wal-dir" default:"/var/log/dicedb" description:"the directory to store WAL segments"`
+	WALDir                            string `mapstructure:"wal-dir" default:"logs" description:"the directory to store WAL segments"`
 	WALMode                           string `mapstructure:"wal-mode" default:"buffered" description:"wal mode to use, values: buffered, unbuffered"`
 	WALWriteMode                      string `mapstructure:"wal-write-mode" default:"default" description:"wal file write mode to use, values: default, fsync"`
 	WALBufferSizeMB                   int    `mapstructure:"wal-buffer-size-mb" default:"1" description:"the size of the wal write buffer in megabytes"`

go.mod

@@ -35,7 +35,7 @@ require (
 	github.com/bytedance/sonic v1.13.2
 	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da
-	github.com/dicedb/dicedb-go v1.0.9
+	github.com/dicedb/dicedb-go v1.0.10
 	github.com/gobwas/glob v0.2.3
 	github.com/google/btree v1.1.3
 	github.com/google/go-cmp v0.6.0

go.sum

@@ -20,8 +20,8 @@ github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa5
 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140 h1:y7y0Oa6UawqTFPCDw9JG6pdKt4F9pAhHv0B7FMGaGD0=
 github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
-github.com/dicedb/dicedb-go v1.0.9 h1:WtXtX6Bu/KFjhnOIlnoo9Z1hbmOtL2tMrUmS6793/Pw=
-github.com/dicedb/dicedb-go v1.0.9/go.mod h1:V1fiCJnPfSObKWrOJ/zrhHEGlLwT9k3pKCto3sz1oW8=
+github.com/dicedb/dicedb-go v1.0.10 h1:inKRSmzpXp7M4vleOfJUV4woISCUFVceDckeum461aw=
+github.com/dicedb/dicedb-go v1.0.10/go.mod h1:V1fiCJnPfSObKWrOJ/zrhHEGlLwT9k3pKCto3sz1oW8=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=

internal/server/ironhawk/iothread.go

@@ -9,7 +9,6 @@ import (
 	"strings"
 
 	"github.com/dicedb/dicedb-go"
-	"google.golang.org/protobuf/proto"
 
 	"github.com/dicedb/dice/config"
 	"github.com/dicedb/dice/internal/auth"
@@ -85,8 +84,7 @@ func (t *IOThread) Start(ctx context.Context, shardManager *shardmanager.ShardMa
 		// Log command to WAL if enabled and not a replay
 		if err == nil && wal.GetWAL() != nil && !_c.IsReplay {
 			// Create WAL entry using protobuf message
-			cmdBytes, _ := proto.Marshal(_c.C)
-			if err := wal.GetWAL().Log(cmdBytes); err != nil {
+			if err := wal.GetWAL().LogCommand(_c.C); err != nil {
 				slog.Error("failed to log command to WAL", slog.Any("error", err))
 			}
 		}

internal/server/ironhawk/main.go

@@ -28,7 +28,7 @@ type Server struct {
 	ioThreadManager *IOThreadManager
 }
 
-func NewServer(shardManager *shardmanager.ShardManager, ioThreadManager *IOThreadManager, watchManager *WatchManager, wl wal.AbstractWAL) *Server {
+func NewServer(shardManager *shardmanager.ShardManager, ioThreadManager *IOThreadManager, watchManager *WatchManager, wl wal.WAL) *Server {
 	return &Server{
 		Host:            config.Config.Host,
 		Port:            config.Config.Port,

internal/wal/wal.go

@@ -5,51 +5,48 @@ package wal
 
 import (
 	"log/slog"
-	sync "sync"
+	"sync"
 	"time"
+
+	w "github.com/dicedb/dicedb-go/wal"
+	"github.com/dicedb/dicedb-go/wire"
 )
 
-type AbstractWAL interface {
-	Log([]byte) error
-	Close() error
+type WAL interface {
 	Init(t time.Time) error
-	Replay(c func(any) error) error
-	Iterate(e any, c func(any) error) error
-}
-
-type WALCmdEntry struct {
-	Command  string   // The command being executed
-	Args     []string // Additional command arguments
-	ClientID string
+	LogCommand(c *wire.Command) error
+	Close() error
+	Replay(c func(*w.Element) error) error
+	Iterate(e *w.Element, c func(*w.Element) error) error
 }
 
 var (
-	ticker   *time.Ticker
-	stopCh   chan struct{}
-	mu       sync.Mutex
-	instance AbstractWAL
+	ticker *time.Ticker
+	stopCh chan struct{}
+	mu     sync.Mutex
+	wl     WAL
 )
 
 // GetWAL returns the global WAL instance
-func GetWAL() AbstractWAL {
+func GetWAL() WAL {
 	mu.Lock()
 	defer mu.Unlock()
-	return instance
+	return wl
 }
 
 // SetGlobalWAL sets the global WAL instance
-func SetWAL(wal AbstractWAL) {
+func SetWAL(_wl WAL) {
 	mu.Lock()
 	defer mu.Unlock()
-	instance = wal
+	wl = _wl
 }
 
 func init() {
 	ticker = time.NewTicker(10 * time.Second)
 	stopCh = make(chan struct{})
 }
 
-func rotateWAL(wl AbstractWAL) {
+func rotateWAL(wl WAL) {
 	mu.Lock()
 	defer mu.Unlock()
 
@@ -62,7 +59,7 @@ func rotateWAL(wl AbstractWAL) {
 	}
 }
 
-func periodicRotate(wl AbstractWAL) {
+func periodicRotate(wl WAL) {
 	for {
 		select {
 		case <-ticker.C:
@@ -73,7 +70,7 @@ func periodicRotate(wl AbstractWAL) {
 	}
 }
 
-func InitBG(wl AbstractWAL) {
+func InitBG(wl WAL) {
 	go periodicRotate(wl)
 }