pk/cosmium
1
0
Fork 0
mirror of https://github.com/pikami/cosmium.git synced 2025-04-15 22:27:58 +01:00
Code Issues Packages Projects Releases Wiki Activity

Use msgpack instead of gob; Added data persistance for badger data store

Browse Source
This commit is contained in:
Pijus Kamandulis 2025-03-13 23:59:07 +02:00
parent 5fe60d831a
commit 97eea30c97
8 changed files with 43 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
api/config/config.go
View File

@ -77,11 +77,22 @@ func (c *ServerConfig) PopulateCalculatedFields() {
logger.SetLogLevel(logger.LogLevelInfo)
}
if c.DataStore == DataStoreBadger &&
(c.InitialDataFilePath != "" || c.PersistDataFilePath != "") {
logger.ErrorLn("InitialData and Persist options are currently not supported with Badger data store")
c.InitialDataFilePath = ""
c.PersistDataFilePath = ""
if c.PersistDataFilePath != "" {
fileInfo, _ := os.Stat(c.PersistDataFilePath)
if c.DataStore == DataStoreMap && fileInfo.IsDir() {
logger.ErrorLn("--Persist cannot be a directory when using default data store")
os.Exit(1)
}
if c.DataStore == DataStoreBadger && !fileInfo.IsDir() {
logger.ErrorLn("--Persist must be a directory when using Badger data store")
os.Exit(1)
}
}
if c.DataStore == DataStoreBadger && c.InitialDataFilePath != "" {
logger.ErrorLn("InitialData option is currently not supported with Badger data store")
os.Exit(1)
}
}

4
cmd/server/server.go
View File

@ -19,7 +19,9 @@ func main() {
var dataStore datastore.DataStore
switch configuration.DataStore {
case config.DataStoreBadger:
dataStore = badgerdatastore.NewBadgerDataStore()
dataStore = badgerdatastore.NewBadgerDataStore(badgerdatastore.BadgerDataStoreOptions{
PersistDataFilePath: configuration.PersistDataFilePath,
})
logger.InfoLn("Using Badger data store")
default:
dataStore = mapdatastore.NewMapDataStore(mapdatastore.MapDataStoreOptions{

2
go.mod
View File

@ -44,6 +44,8 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel v1.35.0 // indirect
go.opentelemetry.io/otel/metric v1.35.0 // indirect

4
go.sum
View File

@ -112,6 +112,10 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=

13
internal/datastore/badger_datastore/badger_datastore.go
View File

@ -1,8 +1,6 @@
package badgerdatastore
import (
"encoding/gob"
"github.com/dgraph-io/badger/v4"
"github.com/pikami/cosmium/internal/logger"
)
@ -11,10 +9,15 @@ type BadgerDataStore struct {
db *badger.DB
}
func NewBadgerDataStore() *BadgerDataStore {
gob.Register([]interface{}{})
type BadgerDataStoreOptions struct {
PersistDataFilePath string
}
badgerOpts := badger.DefaultOptions("").WithInMemory(true)
func NewBadgerDataStore(options BadgerDataStoreOptions) *BadgerDataStore {
badgerOpts := badger.DefaultOptions(options.PersistDataFilePath)
if options.PersistDataFilePath == "" {
badgerOpts = badgerOpts.WithInMemory(true)
}
db, err := badger.Open(badgerOpts)
if err != nil {

13
internal/datastore/badger_datastore/db_abstractions.go
View File

@ -1,13 +1,11 @@
package badgerdatastore
import (
"bytes"
"encoding/gob"
"github.com/dgraph-io/badger/v4"
"github.com/pikami/cosmium/internal/datastore"
"github.com/pikami/cosmium/internal/logger"
"github.com/pikami/cosmium/internal/resourceid"
"github.com/vmihailenco/msgpack/v5"
)
const (
@ -92,14 +90,13 @@ func insertKey(txn *badger.Txn, key string, value interface{}) datastore.DataSto
return datastore.Unknown
}
var buf bytes.Buffer
err = gob.NewEncoder(&buf).Encode(value)
buf, err := msgpack.Marshal(value)
if err != nil {
logger.ErrorLn("Error while encoding value:", err)
return datastore.Unknown
}
err = txn.Set([]byte(key), buf.Bytes())
err = txn.Set([]byte(key), buf)
if err != nil {
logger.ErrorLn("Error while setting key:", err)
return datastore.Unknown
@ -135,7 +132,7 @@ func getKey(txn *badger.Txn, key string, value interface{}) datastore.DataStoreS
return datastore.Unknown
}
err = gob.NewDecoder(bytes.NewReader(val)).Decode(value)
err = msgpack.Unmarshal(val, &value)
if err != nil {
logger.ErrorLn("Error while decoding value:", err)
return datastore.Unknown
@ -158,7 +155,7 @@ func keyExists(txn *badger.Txn, key string) (bool, error) {
}
func listByPrefix[T any](db *badger.DB, prefix string) ([]T, datastore.DataStoreStatus) {
var results []T
results := make([]T, 0)
err := db.View(func(txn *badger.Txn) error {
opts := badger.DefaultIteratorOptions

6
internal/datastore/badger_datastore/document_iterator.go
View File

@ -1,12 +1,10 @@
package badgerdatastore
import (
"bytes"
"encoding/gob"
"github.com/dgraph-io/badger/v4"
"github.com/pikami/cosmium/internal/datastore"
"github.com/pikami/cosmium/internal/logger"
"github.com/vmihailenco/msgpack/v5"
)
type BadgerDocumentIterator struct {
@ -43,7 +41,7 @@ func (i *BadgerDocumentIterator) Next() (datastore.Document, datastore.DataStore
}
current := &datastore.Document{}
err = gob.NewDecoder(bytes.NewReader(val)).Decode(current)
err = msgpack.Unmarshal(val, &current)
if err != nil {
logger.ErrorLn("Error while decoding value:", err)
return datastore.Document{}, datastore.Unknown

4
sharedlibrary/sharedlibrary.go
View File

@ -37,7 +37,9 @@ func CreateServerInstance(serverName *C.char, configurationJSON *C.char) int {
var dataStore datastore.DataStore
switch configuration.DataStore {
case config.DataStoreBadger:
dataStore = badgerdatastore.NewBadgerDataStore()
dataStore = badgerdatastore.NewBadgerDataStore(badgerdatastore.BadgerDataStoreOptions{
PersistDataFilePath: configuration.PersistDataFilePath,
})
default:
dataStore = mapdatastore.NewMapDataStore(mapdatastore.MapDataStoreOptions{
InitialDataFilePath: configuration.InitialDataFilePath,