From 97eea30c97d606c37f30693c9a0f3d4c0fd5bd1c Mon Sep 17 00:00:00 2001 From: Pijus Kamandulis Date: Thu, 13 Mar 2025 23:59:07 +0200 Subject: [PATCH] Use msgpack instead of gob; Added data persistance for badger data store --- api/config/config.go | 21 ++++++++++++++----- cmd/server/server.go | 4 +++- go.mod | 2 ++ go.sum | 4 ++++ .../badger_datastore/badger_datastore.go | 13 +++++++----- .../badger_datastore/db_abstractions.go | 13 +++++------- .../badger_datastore/document_iterator.go | 6 ++---- sharedlibrary/sharedlibrary.go | 4 +++- 8 files changed, 43 insertions(+), 24 deletions(-) diff --git a/api/config/config.go b/api/config/config.go index 984ea41..787b978 100644 --- a/api/config/config.go +++ b/api/config/config.go @@ -77,11 +77,22 @@ func (c *ServerConfig) PopulateCalculatedFields() { logger.SetLogLevel(logger.LogLevelInfo) } - if c.DataStore == DataStoreBadger && - (c.InitialDataFilePath != "" || c.PersistDataFilePath != "") { - logger.ErrorLn("InitialData and Persist options are currently not supported with Badger data store") - c.InitialDataFilePath = "" - c.PersistDataFilePath = "" + if c.PersistDataFilePath != "" { + fileInfo, _ := os.Stat(c.PersistDataFilePath) + if c.DataStore == DataStoreMap && fileInfo.IsDir() { + logger.ErrorLn("--Persist cannot be a directory when using default data store") + os.Exit(1) + } + + if c.DataStore == DataStoreBadger && !fileInfo.IsDir() { + logger.ErrorLn("--Persist must be a directory when using Badger data store") + os.Exit(1) + } + } + + if c.DataStore == DataStoreBadger && c.InitialDataFilePath != "" { + logger.ErrorLn("InitialData option is currently not supported with Badger data store") + os.Exit(1) } } diff --git a/cmd/server/server.go b/cmd/server/server.go index ed49b25..3c47440 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -19,7 +19,9 @@ func main() { var dataStore datastore.DataStore switch configuration.DataStore { case config.DataStoreBadger: - dataStore = badgerdatastore.NewBadgerDataStore() + dataStore = badgerdatastore.NewBadgerDataStore(badgerdatastore.BadgerDataStoreOptions{ + PersistDataFilePath: configuration.PersistDataFilePath, + }) logger.InfoLn("Using Badger data store") default: dataStore = mapdatastore.NewMapDataStore(mapdatastore.MapDataStoreOptions{ diff --git a/go.mod b/go.mod index e02c1a7..139abea 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,8 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect + github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel v1.35.0 // indirect go.opentelemetry.io/otel/metric v1.35.0 // indirect diff --git a/go.sum b/go.sum index 6139959..97b544c 100644 --- a/go.sum +++ b/go.sum @@ -112,6 +112,10 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= diff --git a/internal/datastore/badger_datastore/badger_datastore.go b/internal/datastore/badger_datastore/badger_datastore.go index 690c6db..67664be 100644 --- a/internal/datastore/badger_datastore/badger_datastore.go +++ b/internal/datastore/badger_datastore/badger_datastore.go @@ -1,8 +1,6 @@ package badgerdatastore import ( - "encoding/gob" - "github.com/dgraph-io/badger/v4" "github.com/pikami/cosmium/internal/logger" ) @@ -11,10 +9,15 @@ type BadgerDataStore struct { db *badger.DB } -func NewBadgerDataStore() *BadgerDataStore { - gob.Register([]interface{}{}) +type BadgerDataStoreOptions struct { + PersistDataFilePath string +} - badgerOpts := badger.DefaultOptions("").WithInMemory(true) +func NewBadgerDataStore(options BadgerDataStoreOptions) *BadgerDataStore { + badgerOpts := badger.DefaultOptions(options.PersistDataFilePath) + if options.PersistDataFilePath == "" { + badgerOpts = badgerOpts.WithInMemory(true) + } db, err := badger.Open(badgerOpts) if err != nil { diff --git a/internal/datastore/badger_datastore/db_abstractions.go b/internal/datastore/badger_datastore/db_abstractions.go index fdc5aa6..8dc2cf9 100644 --- a/internal/datastore/badger_datastore/db_abstractions.go +++ b/internal/datastore/badger_datastore/db_abstractions.go @@ -1,13 +1,11 @@ package badgerdatastore import ( - "bytes" - "encoding/gob" - "github.com/dgraph-io/badger/v4" "github.com/pikami/cosmium/internal/datastore" "github.com/pikami/cosmium/internal/logger" "github.com/pikami/cosmium/internal/resourceid" + "github.com/vmihailenco/msgpack/v5" ) const ( @@ -92,14 +90,13 @@ func insertKey(txn *badger.Txn, key string, value interface{}) datastore.DataSto return datastore.Unknown } - var buf bytes.Buffer - err = gob.NewEncoder(&buf).Encode(value) + buf, err := msgpack.Marshal(value) if err != nil { logger.ErrorLn("Error while encoding value:", err) return datastore.Unknown } - err = txn.Set([]byte(key), buf.Bytes()) + err = txn.Set([]byte(key), buf) if err != nil { logger.ErrorLn("Error while setting key:", err) return datastore.Unknown @@ -135,7 +132,7 @@ func getKey(txn *badger.Txn, key string, value interface{}) datastore.DataStoreS return datastore.Unknown } - err = gob.NewDecoder(bytes.NewReader(val)).Decode(value) + err = msgpack.Unmarshal(val, &value) if err != nil { logger.ErrorLn("Error while decoding value:", err) return datastore.Unknown @@ -158,7 +155,7 @@ func keyExists(txn *badger.Txn, key string) (bool, error) { } func listByPrefix[T any](db *badger.DB, prefix string) ([]T, datastore.DataStoreStatus) { - var results []T + results := make([]T, 0) err := db.View(func(txn *badger.Txn) error { opts := badger.DefaultIteratorOptions diff --git a/internal/datastore/badger_datastore/document_iterator.go b/internal/datastore/badger_datastore/document_iterator.go index 6fe9172..0cf5d16 100644 --- a/internal/datastore/badger_datastore/document_iterator.go +++ b/internal/datastore/badger_datastore/document_iterator.go @@ -1,12 +1,10 @@ package badgerdatastore import ( - "bytes" - "encoding/gob" - "github.com/dgraph-io/badger/v4" "github.com/pikami/cosmium/internal/datastore" "github.com/pikami/cosmium/internal/logger" + "github.com/vmihailenco/msgpack/v5" ) type BadgerDocumentIterator struct { @@ -43,7 +41,7 @@ func (i *BadgerDocumentIterator) Next() (datastore.Document, datastore.DataStore } current := &datastore.Document{} - err = gob.NewDecoder(bytes.NewReader(val)).Decode(current) + err = msgpack.Unmarshal(val, ¤t) if err != nil { logger.ErrorLn("Error while decoding value:", err) return datastore.Document{}, datastore.Unknown diff --git a/sharedlibrary/sharedlibrary.go b/sharedlibrary/sharedlibrary.go index 5d69387..beac3e4 100644 --- a/sharedlibrary/sharedlibrary.go +++ b/sharedlibrary/sharedlibrary.go @@ -37,7 +37,9 @@ func CreateServerInstance(serverName *C.char, configurationJSON *C.char) int { var dataStore datastore.DataStore switch configuration.DataStore { case config.DataStoreBadger: - dataStore = badgerdatastore.NewBadgerDataStore() + dataStore = badgerdatastore.NewBadgerDataStore(badgerdatastore.BadgerDataStoreOptions{ + PersistDataFilePath: configuration.PersistDataFilePath, + }) default: dataStore = mapdatastore.NewMapDataStore(mapdatastore.MapDataStoreOptions{ InitialDataFilePath: configuration.InitialDataFilePath,