Use msgpack instead of gob; Added data persistance for badger data store

api/config/config.go

@@ -77,11 +77,22 @@ func (c *ServerConfig) PopulateCalculatedFields() {
 		logger.SetLogLevel(logger.LogLevelInfo)
 	}
 
-	if c.DataStore == DataStoreBadger &&
+	if c.PersistDataFilePath != "" {
-		(c.InitialDataFilePath != "" || c.PersistDataFilePath != "") {
+		fileInfo, _ := os.Stat(c.PersistDataFilePath)
-		logger.ErrorLn("InitialData and Persist options are currently not supported with Badger data store")
+		if c.DataStore == DataStoreMap && fileInfo.IsDir() {
-		c.InitialDataFilePath = ""
+			logger.ErrorLn("--Persist cannot be a directory when using default data store")
-		c.PersistDataFilePath = ""
+			os.Exit(1)
+		}
+
+		if c.DataStore == DataStoreBadger && !fileInfo.IsDir() {
+			logger.ErrorLn("--Persist must be a directory when using Badger data store")
+			os.Exit(1)
+		}
+	}
+
+	if c.DataStore == DataStoreBadger && c.InitialDataFilePath != "" {
+		logger.ErrorLn("InitialData option is currently not supported with Badger data store")
+		os.Exit(1)
 	}
 }
 

cmd/server/server.go

@@ -19,7 +19,9 @@ func main() {
 	var dataStore datastore.DataStore
 	switch configuration.DataStore {
 	case config.DataStoreBadger:
-		dataStore = badgerdatastore.NewBadgerDataStore()
+		dataStore = badgerdatastore.NewBadgerDataStore(badgerdatastore.BadgerDataStoreOptions{
+			PersistDataFilePath: configuration.PersistDataFilePath,
+		})
 		logger.InfoLn("Using Badger data store")
 	default:
 		dataStore = mapdatastore.NewMapDataStore(mapdatastore.MapDataStoreOptions{

go.mod

@@ -44,6 +44,8 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/otel v1.35.0 // indirect
 	go.opentelemetry.io/otel/metric v1.35.0 // indirect

go.sum

@@ -112,6 +112,10 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
 github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
+github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
+github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
+github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
 go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
 go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
 go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=

internal/datastore/badger_datastore/badger_datastore.go

@@ -1,8 +1,6 @@
 package badgerdatastore
 
 import (
-	"encoding/gob"
-
 	"github.com/dgraph-io/badger/v4"
 	"github.com/pikami/cosmium/internal/logger"
 )
@@ -11,10 +9,15 @@ type BadgerDataStore struct {
 	db *badger.DB
 }
 
-func NewBadgerDataStore() *BadgerDataStore {
+type BadgerDataStoreOptions struct {
-	gob.Register([]interface{}{})
+	PersistDataFilePath string
+}
 
-	badgerOpts := badger.DefaultOptions("").WithInMemory(true)
+func NewBadgerDataStore(options BadgerDataStoreOptions) *BadgerDataStore {
+	badgerOpts := badger.DefaultOptions(options.PersistDataFilePath)
+	if options.PersistDataFilePath == "" {
+		badgerOpts = badgerOpts.WithInMemory(true)
+	}
 
 	db, err := badger.Open(badgerOpts)
 	if err != nil {

internal/datastore/badger_datastore/db_abstractions.go

@@ -1,13 +1,11 @@
 package badgerdatastore
 
 import (
-	"bytes"
-	"encoding/gob"
-
 	"github.com/dgraph-io/badger/v4"
 	"github.com/pikami/cosmium/internal/datastore"
 	"github.com/pikami/cosmium/internal/logger"
 	"github.com/pikami/cosmium/internal/resourceid"
+	"github.com/vmihailenco/msgpack/v5"
 )
 
 const (
@@ -92,14 +90,13 @@ func insertKey(txn *badger.Txn, key string, value interface{}) datastore.DataSto
 		return datastore.Unknown
 	}
 
-	var buf bytes.Buffer
+	buf, err := msgpack.Marshal(value)
-	err = gob.NewEncoder(&buf).Encode(value)
 	if err != nil {
 		logger.ErrorLn("Error while encoding value:", err)
 		return datastore.Unknown
 	}
 
-	err = txn.Set([]byte(key), buf.Bytes())
+	err = txn.Set([]byte(key), buf)
 	if err != nil {
 		logger.ErrorLn("Error while setting key:", err)
 		return datastore.Unknown
@@ -135,7 +132,7 @@ func getKey(txn *badger.Txn, key string, value interface{}) datastore.DataStoreS
 		return datastore.Unknown
 	}
 
-	err = gob.NewDecoder(bytes.NewReader(val)).Decode(value)
+	err = msgpack.Unmarshal(val, &value)
 	if err != nil {
 		logger.ErrorLn("Error while decoding value:", err)
 		return datastore.Unknown
@@ -158,7 +155,7 @@ func keyExists(txn *badger.Txn, key string) (bool, error) {
 }
 
 func listByPrefix[T any](db *badger.DB, prefix string) ([]T, datastore.DataStoreStatus) {
-	var results []T
+	results := make([]T, 0)
 
 	err := db.View(func(txn *badger.Txn) error {
 		opts := badger.DefaultIteratorOptions

internal/datastore/badger_datastore/document_iterator.go

@@ -1,12 +1,10 @@
 package badgerdatastore
 
 import (
-	"bytes"
-	"encoding/gob"
-
 	"github.com/dgraph-io/badger/v4"
 	"github.com/pikami/cosmium/internal/datastore"
 	"github.com/pikami/cosmium/internal/logger"
+	"github.com/vmihailenco/msgpack/v5"
 )
 
 type BadgerDocumentIterator struct {
@@ -43,7 +41,7 @@ func (i *BadgerDocumentIterator) Next() (datastore.Document, datastore.DataStore
 	}
 
 	current := &datastore.Document{}
-	err = gob.NewDecoder(bytes.NewReader(val)).Decode(current)
+	err = msgpack.Unmarshal(val, &current)
 	if err != nil {
 		logger.ErrorLn("Error while decoding value:", err)
 		return datastore.Document{}, datastore.Unknown

sharedlibrary/sharedlibrary.go

@@ -37,7 +37,9 @@ func CreateServerInstance(serverName *C.char, configurationJSON *C.char) int {
 	var dataStore datastore.DataStore
 	switch configuration.DataStore {
 	case config.DataStoreBadger:
-		dataStore = badgerdatastore.NewBadgerDataStore()
+		dataStore = badgerdatastore.NewBadgerDataStore(badgerdatastore.BadgerDataStoreOptions{
+			PersistDataFilePath: configuration.PersistDataFilePath,
+		})
 	default:
 		dataStore = mapdatastore.NewMapDataStore(mapdatastore.MapDataStoreOptions{
 			InitialDataFilePath: configuration.InitialDataFilePath,