From 1b84f304d17c1311e122d9a035a7a1bfcd312d4c Mon Sep 17 00:00:00 2001 From: Vincent Jordan Date: Fri, 9 Feb 2024 13:38:36 +0100 Subject: [PATCH 1/6] feat(storers): Add a new NutsMemcached storage backend built on NutsDB In this commit NutsMemcached behave just like Nuts. --- configurationtypes/types.go | 7 + pkg/storage/nutsMemcachedProvider.go | 273 +++++++++++++++++++++++++++ pkg/storage/storage.go | 3 + plugins/caddy/configuration.go | 25 +++ 4 files changed, 308 insertions(+) create mode 100644 pkg/storage/nutsMemcachedProvider.go diff --git a/configurationtypes/types.go b/configurationtypes/types.go index 884df845f..3bf6e52f5 100644 --- a/configurationtypes/types.go +++ b/configurationtypes/types.go @@ -232,6 +232,7 @@ type DefaultCache struct { Etcd CacheProvider `json:"etcd" yaml:"etcd"` Mode string `json:"mode" yaml:"mode"` Nuts CacheProvider `json:"nuts" yaml:"nuts"` + NutsMemcached CacheProvider `json:"nuts_memcached" yaml:"nuts_memcached"` Olric CacheProvider `json:"olric" yaml:"olric"` Otter CacheProvider `json:"otter" yaml:"otter"` Redis CacheProvider `json:"redis" yaml:"redis"` @@ -300,6 +301,11 @@ func (d *DefaultCache) GetOtter() CacheProvider { return d.Otter } +// GetNutsMemcached returns nuts_memcached configuration +func (d *DefaultCache) GetNutsMemcached() CacheProvider { + return d.NutsMemcached +} + // GetOlric returns olric configuration func (d *DefaultCache) GetOlric() CacheProvider { return d.Olric @@ -356,6 +362,7 @@ type DefaultCacheInterface interface { GetMode() string GetOtter() CacheProvider GetNuts() CacheProvider + GetNutsMemcached() CacheProvider GetOlric() CacheProvider GetRedis() CacheProvider GetHeaders() []string diff --git a/pkg/storage/nutsMemcachedProvider.go b/pkg/storage/nutsMemcachedProvider.go new file mode 100644 index 000000000..d6489be75 --- /dev/null +++ b/pkg/storage/nutsMemcachedProvider.go @@ -0,0 +1,273 @@ +package storage + +import ( + "bufio" + "bytes" + "encoding/json" + "net/http" + "strings" + "time" + + t "github.com/darkweak/souin/configurationtypes" + "github.com/darkweak/souin/pkg/rfc" + "github.com/darkweak/souin/pkg/storage/types" + "github.com/imdario/mergo" + "github.com/nutsdb/nutsdb" + "go.uber.org/zap" +) + +var nutsMemcachedInstanceMap = map[string]*nutsdb.DB{} + +// NutsMemcached provider type +type NutsMemcached struct { + *nutsdb.DB + stale time.Duration + logger *zap.Logger +} + +// const ( +// bucket = "souin-bucket" +// nutsLimit = 1 << 16 +// ) + +// func sanitizeProperties(m map[string]interface{}) map[string]interface{} { +// iotas := []string{"RWMode", "StartFileLoadingMode"} +// for _, i := range iotas { +// if v := m[i]; v != nil { +// currentMode := nutsdb.FileIO +// switch v { +// case 1: +// currentMode = nutsdb.MMap +// } +// m[i] = currentMode +// } +// } + +// for _, i := range []string{"SegmentSize", "NodeNum", "MaxFdNumsInCache"} { +// if v := m[i]; v != nil { +// m[i], _ = v.(int64) +// } +// } + +// if v := m["EntryIdxMode"]; v != nil { +// m["EntryIdxMode"] = nutsdb.HintKeyValAndRAMIdxMode +// switch v { +// case 1: +// m["EntryIdxMode"] = nutsdb.HintKeyAndRAMIdxMode +// } +// } + +// if v := m["SyncEnable"]; v != nil { +// m["SyncEnable"] = true +// if b, ok := v.(bool); ok { +// m["SyncEnable"] = b +// } else if s, ok := v.(string); ok { +// m["SyncEnable"], _ = strconv.ParseBool(s) +// } +// } + +// return m +// } + +// NutsConnectionFactory function create new Nuts instance +func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.Storer, error) { + dc := c.GetDefaultCache() + nutsConfiguration := dc.GetNutsMemcached() + nutsOptions := nutsdb.DefaultOptions + nutsOptions.Dir = "/tmp/souin-nuts-memcached" + if nutsConfiguration.Configuration != nil { + var parsedNuts nutsdb.Options + nutsConfiguration.Configuration = sanitizeProperties(nutsConfiguration.Configuration.(map[string]interface{})) + if b, e := json.Marshal(nutsConfiguration.Configuration); e == nil { + if e = json.Unmarshal(b, &parsedNuts); e != nil { + c.GetLogger().Sugar().Error("Impossible to parse the configuration for the Nuts provider", e) + } + } + + if err := mergo.Merge(&nutsOptions, parsedNuts, mergo.WithOverride); err != nil { + c.GetLogger().Sugar().Error("An error occurred during the nutsOptions merge from the default options with your configuration.") + } + } else { + nutsOptions.RWMode = nutsdb.MMap + if nutsConfiguration.Path != "" { + nutsOptions.Dir = nutsConfiguration.Path + } + } + + if instance, ok := nutsMemcachedInstanceMap[nutsOptions.Dir]; ok && instance != nil { + return &NutsMemcached{ + DB: instance, + stale: dc.GetStale(), + logger: c.GetLogger(), + }, nil + } + + db, e := nutsdb.Open(nutsOptions) + + if e != nil { + c.GetLogger().Sugar().Error("Impossible to open the Nuts DB.", e) + return nil, e + } + + instance := &NutsMemcached{ + DB: db, + stale: dc.GetStale(), + logger: c.GetLogger(), + } + nutsMemcachedInstanceMap[nutsOptions.Dir] = instance.DB + + return instance, nil +} + +// Name returns the storer name +func (provider *NutsMemcached) Name() string { + return "NUTS_MEMCACHED" +} + +// ListKeys method returns the list of existing keys +func (provider *NutsMemcached) ListKeys() []string { + keys := []string{} + + e := provider.DB.View(func(tx *nutsdb.Tx) error { + e, _ := tx.GetAll(bucket) + for _, k := range e { + if !strings.Contains(string(k.Key), surrogatePrefix) { + keys = append(keys, string(k.Key)) + } + } + return nil + }) + + if e != nil { + return []string{} + } + + return keys +} + +// MapKeys method returns the map of existing keys +func (provider *NutsMemcached) MapKeys(prefix string) map[string]string { + keys := map[string]string{} + + e := provider.DB.View(func(tx *nutsdb.Tx) error { + e, _ := tx.GetAll(bucket) + for _, k := range e { + if strings.HasPrefix(string(k.Key), prefix) { + nk, _ := strings.CutPrefix(string(k.Key), prefix) + keys[nk] = string(k.Value) + } + } + return nil + }) + + if e != nil { + return map[string]string{} + } + + return keys +} + +// Get method returns the populated response if exists, empty response then +func (provider *NutsMemcached) Get(key string) (item []byte) { + _ = provider.DB.View(func(tx *nutsdb.Tx) error { + i, e := tx.Get(bucket, []byte(key)) + if i != nil { + item = i.Value + } + return e + }) + + return +} + +// Prefix method returns the populated response if exists, empty response then +func (provider *NutsMemcached) Prefix(key string, req *http.Request, validator *rfc.Revalidator) *http.Response { + var result *http.Response + + _ = provider.DB.View(func(tx *nutsdb.Tx) error { + prefix := []byte(key) + + if entries, err := tx.PrefixSearchScan(bucket, prefix, "^({|$)", 0, 50); err != nil { + return err + } else { + for _, entry := range entries { + if varyVoter(key, req, string(entry.Key)) { + if res, err := http.ReadResponse(bufio.NewReader(bytes.NewBuffer(entry.Value)), req); err == nil { + rfc.ValidateETag(res, validator) + if validator.Matched { + provider.logger.Sugar().Debugf("The stored key %s matched the current iteration key ETag %+v", string(entry.Key), validator) + result = res + return nil + } + + provider.logger.Sugar().Debugf("The stored key %s didn't match the current iteration key ETag %+v", string(entry.Key), validator) + } else { + provider.logger.Sugar().Errorf("An error occured while reading response for the key %s: %v", string(entry.Key), err) + } + } + } + } + return nil + }) + + return result +} + +// Set method will store the response in Nuts provider +func (provider *NutsMemcached) Set(key string, value []byte, url t.URL, duration time.Duration) error { + if duration == 0 { + duration = url.TTL.Duration + } + + err := provider.DB.Update(func(tx *nutsdb.Tx) error { + return tx.Put(bucket, []byte(key), value, uint32(duration.Seconds())) + }) + + if err != nil { + provider.logger.Sugar().Errorf("Impossible to set value into Nuts, %v", err) + return err + } + + err = provider.DB.Update(func(tx *nutsdb.Tx) error { + return tx.Put(bucket, []byte(StalePrefix+key), value, uint32((provider.stale + duration).Seconds())) + }) + + if err != nil { + provider.logger.Sugar().Errorf("Impossible to set value into Nuts, %v", err) + } + + return nil +} + +// Delete method will delete the response in Nuts provider if exists corresponding to key param +func (provider *NutsMemcached) Delete(key string) { + _ = provider.DB.Update(func(tx *nutsdb.Tx) error { + return tx.Delete(bucket, []byte(key)) + }) +} + +// DeleteMany method will delete the responses in Nuts provider if exists corresponding to the regex key param +func (provider *NutsMemcached) DeleteMany(key string) { + _ = provider.DB.Update(func(tx *nutsdb.Tx) error { + if entries, err := tx.PrefixSearchScan(bucket, []byte(""), key, 0, nutsLimit); err != nil { + return err + } else { + for _, entry := range entries { + _ = tx.Delete(bucket, entry.Key) + } + } + return nil + }) +} + +// Init method will +func (provider *NutsMemcached) Init() error { + return nil +} + +// Reset method will reset or close provider +func (provider *NutsMemcached) Reset() error { + return provider.DB.Update(func(tx *nutsdb.Tx) error { + return tx.DeleteBucket(1, bucket) + }) +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index d056385aa..8c90cb4f8 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -34,6 +34,7 @@ var storageMap = map[string]StorerInstanciator{ "otter": OtterConnectionFactory, "embedded_olric": EmbeddedOlricConnectionFactory, "nuts": NutsConnectionFactory, + "nuts_memcached": NutsMemcachedConnectionFactory, "badger": BadgerConnectionFactory, } @@ -54,6 +55,8 @@ func getStorageNameFromConfiguration(configuration configurationtypes.AbstractCo return "otter" } else if configuration.GetDefaultCache().GetNuts().Configuration != nil || configuration.GetDefaultCache().GetNuts().Path != "" { return "nuts" + } else if configuration.GetDefaultCache().GetNutsMemcached().Configuration != nil || configuration.GetDefaultCache().GetNutsMemcached().Path != "" { + return "nuts_memcached" } return "badger" diff --git a/plugins/caddy/configuration.go b/plugins/caddy/configuration.go index 2d9c5fc24..726306ac1 100644 --- a/plugins/caddy/configuration.go +++ b/plugins/caddy/configuration.go @@ -42,6 +42,8 @@ type DefaultCache struct { Nuts configurationtypes.CacheProvider `json:"nuts"` // Otter provider configuration. Otter configurationtypes.CacheProvider `json:"otter"` + // NutsMemcached provider configuration. + NutsMemcached configurationtypes.CacheProvider `json:"nuts_memcached"` // Regex to exclude cache. Regex configurationtypes.Regex `json:"regex"` // Storage providers chaining and order. @@ -109,6 +111,11 @@ func (d *DefaultCache) GetOtter() configurationtypes.CacheProvider { return d.Otter } +// GetNutsMemcached returns nuts_memcached configuration +func (d *DefaultCache) GetNutsMemcached() configurationtypes.CacheProvider { + return d.NutsMemcached +} + // GetOlric returns olric configuration func (d *DefaultCache) GetOlric() configurationtypes.CacheProvider { return d.Olric @@ -518,6 +525,24 @@ func parseConfiguration(cfg *Configuration, h *caddyfile.Dispenser, isGlobal boo } } cfg.DefaultCache.Otter = provider + case "nuts_memcached": + provider := configurationtypes.CacheProvider{} + for nesting := h.Nesting(); h.NextBlock(nesting); { + directive := h.Val() + switch directive { + case "url": + urlArgs := h.RemainingArgs() + provider.URL = urlArgs[0] + case "path": + urlArgs := h.RemainingArgs() + provider.Path = urlArgs[0] + case "configuration": + provider.Configuration = parseCaddyfileRecursively(h) + default: + return h.Errf("unsupported nuts directive: %s", directive) + } + } + cfg.DefaultCache.NutsMemcached = provider case "olric": cfg.DefaultCache.Distributed = true provider := configurationtypes.CacheProvider{} From 05bd50704c91a2db9043531f87a05ef46bc5d32e Mon Sep 17 00:00:00 2001 From: Vincent Jordan Date: Mon, 12 Feb 2024 10:03:30 +0100 Subject: [PATCH 2/6] feat(storers): Add memcached backend to NutsMemcached In this commit cache values are moved to memcached and only keys are kept in Nuts. --- go.mod | 1 + go.sum | 2 + pkg/storage/nutsMemcachedProvider.go | 130 ++++++++++++++++++++------- 3 files changed, 100 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 268430fd1..b0eddaf78 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/darkweak/souin go 1.21 require ( + github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 github.com/buraksezer/olric v0.5.4 github.com/cespare/xxhash/v2 v2.2.0 github.com/dgraph-io/badger/v3 v3.2103.5 diff --git a/go.sum b/go.sum index d1f5edd0b..b651fce13 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 h1:N7oVaKyGp8bttX0bfZGmcGkjz7DLQXhAn3DNd3T0ous= +github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c= github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW25rrZlU= github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= github.com/buraksezer/olric v0.5.4 h1:LDgLIfVoyol4qzdNirrrDUKqzFw0yDsa7ukvLrpP4cU= diff --git a/pkg/storage/nutsMemcachedProvider.go b/pkg/storage/nutsMemcachedProvider.go index d6489be75..e11f35c46 100644 --- a/pkg/storage/nutsMemcachedProvider.go +++ b/pkg/storage/nutsMemcachedProvider.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/bradfitz/gomemcache/memcache" t "github.com/darkweak/souin/configurationtypes" "github.com/darkweak/souin/pkg/rfc" "github.com/darkweak/souin/pkg/storage/types" @@ -21,8 +22,9 @@ var nutsMemcachedInstanceMap = map[string]*nutsdb.DB{} // NutsMemcached provider type type NutsMemcached struct { *nutsdb.DB - stale time.Duration - logger *zap.Logger + stale time.Duration + logger *zap.Logger + memcacheClient *memcache.Client } // const ( @@ -75,6 +77,13 @@ func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.S nutsConfiguration := dc.GetNutsMemcached() nutsOptions := nutsdb.DefaultOptions nutsOptions.Dir = "/tmp/souin-nuts-memcached" + + // `HintKeyAndRAMIdxMode` represents ram index (only key) mode. + nutsOptions.EntryIdxMode = nutsdb.HintKeyAndRAMIdxMode + // `HintBPTSparseIdxMode` represents b+ tree sparse index mode. + // Note: this mode was removed after v0.14.0 + //nutsOptions.EntryIdxMode = nutsdb.HintBPTSparseIdxMode + if nutsConfiguration.Configuration != nil { var parsedNuts nutsdb.Options nutsConfiguration.Configuration = sanitizeProperties(nutsConfiguration.Configuration.(map[string]interface{})) @@ -110,9 +119,10 @@ func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.S } instance := &NutsMemcached{ - DB: db, - stale: dc.GetStale(), - logger: c.GetLogger(), + DB: db, + stale: dc.GetStale(), + logger: c.GetLogger(), + memcacheClient: memcache.New("127.0.0.1:11211"), // hardcoded for now } nutsMemcachedInstanceMap[nutsOptions.Dir] = instance.DB @@ -169,13 +179,30 @@ func (provider *NutsMemcached) MapKeys(prefix string) map[string]string { // Get method returns the populated response if exists, empty response then func (provider *NutsMemcached) Get(key string) (item []byte) { - _ = provider.DB.View(func(tx *nutsdb.Tx) error { - i, e := tx.Get(bucket, []byte(key)) - if i != nil { + // get from nuts + keyFound := false + { + _ = provider.DB.View(func(tx *nutsdb.Tx) error { + i, e := tx.Get(bucket, []byte(key)) + if i != nil { + // Value is stored in memcached + //item = i.Value + keyFound = true + } + return e + }) + } + + // get from memcached + if keyFound { + // Reminder: the key must be at most 250 bytes in length + //fmt.Println("memcached GET", key) + i, e := provider.memcacheClient.Get(key) + if e == nil && i != nil { item = i.Value } - return e - }) + + } return } @@ -192,17 +219,29 @@ func (provider *NutsMemcached) Prefix(key string, req *http.Request, validator * } else { for _, entry := range entries { if varyVoter(key, req, string(entry.Key)) { - if res, err := http.ReadResponse(bufio.NewReader(bytes.NewBuffer(entry.Value)), req); err == nil { - rfc.ValidateETag(res, validator) - if validator.Matched { - provider.logger.Sugar().Debugf("The stored key %s matched the current iteration key ETag %+v", string(entry.Key), validator) - result = res - return nil + // TODO: improve this + // store header only in nuts and avoid query to memcached on each vary + // E.g, rfc.ValidateETag on NutsDB header value, retrieve response body later from memcached. + + // Reminder: the key must be at most 250 bytes in length + //fmt.Println("memcached PREFIX", key, "GET", string(entry.Key)) + i, e := provider.memcacheClient.Get(string(entry.Key)) + if e == nil && i != nil { + res, err := http.ReadResponse(bufio.NewReader(bytes.NewBuffer(i.Value)), req) + if err == nil { + rfc.ValidateETag(res, validator) + if validator.Matched { + provider.logger.Sugar().Debugf("The stored key %s matched the current iteration key ETag %+v", string(entry.Key), validator) + result = res + return nil + } + + provider.logger.Sugar().Debugf("The stored key %s didn't match the current iteration key ETag %+v", string(entry.Key), validator) + } else { + provider.logger.Sugar().Errorf("An error occured while reading response for the key %s: %v", string(entry.Key), err) } - - provider.logger.Sugar().Debugf("The stored key %s didn't match the current iteration key ETag %+v", string(entry.Key), validator) } else { - provider.logger.Sugar().Errorf("An error occured while reading response for the key %s: %v", string(entry.Key), err) + provider.logger.Sugar().Errorf("An error occured while reading memcached for the key %s: %v", string(entry.Key), err) } } } @@ -214,26 +253,51 @@ func (provider *NutsMemcached) Prefix(key string, req *http.Request, validator * } // Set method will store the response in Nuts provider -func (provider *NutsMemcached) Set(key string, value []byte, url t.URL, duration time.Duration) error { - if duration == 0 { - duration = url.TTL.Duration +func (provider *NutsMemcached) Set(key string, value []byte, url t.URL, ttl time.Duration) error { + if ttl == 0 { + ttl = url.TTL.Duration } - err := provider.DB.Update(func(tx *nutsdb.Tx) error { - return tx.Put(bucket, []byte(key), value, uint32(duration.Seconds())) - }) + // set to nuts (normal TTL) + { + err := provider.DB.Update(func(tx *nutsdb.Tx) error { + // No value is stored, value is stored in memcached + return tx.Put(bucket, []byte(key), []byte{}, uint32(ttl.Seconds())) + }) - if err != nil { - provider.logger.Sugar().Errorf("Impossible to set value into Nuts, %v", err) - return err + if err != nil { + provider.logger.Sugar().Errorf("Impossible to set value into Nuts, %v", err) + return err + } } - err = provider.DB.Update(func(tx *nutsdb.Tx) error { - return tx.Put(bucket, []byte(StalePrefix+key), value, uint32((provider.stale + duration).Seconds())) - }) + // set to nuts (stale TTL) + staleTtl := int32((provider.stale + ttl).Seconds()) + { + err := provider.DB.Update(func(tx *nutsdb.Tx) error { + // No value is stored, value is stored in memcached + return tx.Put(bucket, []byte(StalePrefix+key), []byte{}, uint32(staleTtl)) + }) - if err != nil { - provider.logger.Sugar().Errorf("Impossible to set value into Nuts, %v", err) + if err != nil { + provider.logger.Sugar().Errorf("Impossible to set value into Nuts, %v", err) + } + } + + // set to memcached with stale TTL + { + // Reminder: the key must be at most 250 bytes in length + //fmt.Println("memcached SET", key) + err := provider.memcacheClient.Set( + &memcache.Item{ + Key: key, + Value: value, + Expiration: staleTtl, + }, + ) + if err != nil { + provider.logger.Sugar().Errorf("Impossible to set value into Memcached, %v", err) + } } return nil From 43bab163881e9022e39fd7aee91bbfd2c3adb265 Mon Sep 17 00:00:00 2001 From: Vincent Jordan Date: Wed, 14 Feb 2024 11:38:42 +0100 Subject: [PATCH 3/6] feat(storers): Switch Memcached to Ristretto --- pkg/storage/nutsMemcachedProvider.go | 174 ++++++++++++++++++--------- 1 file changed, 120 insertions(+), 54 deletions(-) diff --git a/pkg/storage/nutsMemcachedProvider.go b/pkg/storage/nutsMemcachedProvider.go index e11f35c46..0752adb90 100644 --- a/pkg/storage/nutsMemcachedProvider.go +++ b/pkg/storage/nutsMemcachedProvider.go @@ -4,14 +4,16 @@ import ( "bufio" "bytes" "encoding/json" + "errors" "net/http" + "strconv" "strings" "time" - "github.com/bradfitz/gomemcache/memcache" t "github.com/darkweak/souin/configurationtypes" "github.com/darkweak/souin/pkg/rfc" "github.com/darkweak/souin/pkg/storage/types" + "github.com/dgraph-io/ristretto" "github.com/imdario/mergo" "github.com/nutsdb/nutsdb" "go.uber.org/zap" @@ -22,9 +24,10 @@ var nutsMemcachedInstanceMap = map[string]*nutsdb.DB{} // NutsMemcached provider type type NutsMemcached struct { *nutsdb.DB - stale time.Duration - logger *zap.Logger - memcacheClient *memcache.Client + stale time.Duration + logger *zap.Logger + //memcacheClient *memcache.Client + ristrettoCache *ristretto.Cache } // const ( @@ -82,6 +85,7 @@ func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.S nutsOptions.EntryIdxMode = nutsdb.HintKeyAndRAMIdxMode // `HintBPTSparseIdxMode` represents b+ tree sparse index mode. // Note: this mode was removed after v0.14.0 + // Use: github.com/nutsdb/nutsdb v0.14.0 //nutsOptions.EntryIdxMode = nutsdb.HintBPTSparseIdxMode if nutsConfiguration.Configuration != nil { @@ -118,11 +122,35 @@ func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.S return nil, e } + var numCounters int64 = 1e7 // number of keys to track frequency of (10M). + var maxCost int64 = 1 << 30 // maximum cost of cache (1GB). + if nutsConfiguration.Configuration != nil { + rawNumCounters, ok := nutsConfiguration.Configuration.(map[string]interface{})["NumCounters"] + if ok { + numCounters, _ = strconv.ParseInt(rawNumCounters.(string), 10, 64) + } + + rawMaxCost, ok := nutsConfiguration.Configuration.(map[string]interface{})["MaxCost"] + if ok { + maxCost, _ = strconv.ParseInt(rawMaxCost.(string), 10, 64) + } + } + // See https://github.com/dgraph-io/ristretto?tab=readme-ov-file#example + ristrettoCache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: numCounters, + MaxCost: maxCost, + BufferItems: 64, // number of keys per Get buffer. + }) + if err != nil { + panic(err) + } + instance := &NutsMemcached{ - DB: db, - stale: dc.GetStale(), - logger: c.GetLogger(), - memcacheClient: memcache.New("127.0.0.1:11211"), // hardcoded for now + DB: db, + stale: dc.GetStale(), + logger: c.GetLogger(), + //memcacheClient: memcache.New("127.0.0.1:11211"), // hardcoded for now + ristrettoCache: ristrettoCache, } nutsMemcachedInstanceMap[nutsOptions.Dir] = instance.DB @@ -179,29 +207,10 @@ func (provider *NutsMemcached) MapKeys(prefix string) map[string]string { // Get method returns the populated response if exists, empty response then func (provider *NutsMemcached) Get(key string) (item []byte) { - // get from nuts - keyFound := false - { - _ = provider.DB.View(func(tx *nutsdb.Tx) error { - i, e := tx.Get(bucket, []byte(key)) - if i != nil { - // Value is stored in memcached - //item = i.Value - keyFound = true - } - return e - }) - } - - // get from memcached - if keyFound { - // Reminder: the key must be at most 250 bytes in length - //fmt.Println("memcached GET", key) - i, e := provider.memcacheClient.Get(key) - if e == nil && i != nil { - item = i.Value - } + memcachedKey, _ := provider.getFromNuts(key) + if memcachedKey != "" { + item, _ = provider.getFromMemcached(memcachedKey) } return @@ -220,14 +229,14 @@ func (provider *NutsMemcached) Prefix(key string, req *http.Request, validator * for _, entry := range entries { if varyVoter(key, req, string(entry.Key)) { // TODO: improve this - // store header only in nuts and avoid query to memcached on each vary + // Store only response header in nuts and avoid query to memcached on each vary // E.g, rfc.ValidateETag on NutsDB header value, retrieve response body later from memcached. // Reminder: the key must be at most 250 bytes in length //fmt.Println("memcached PREFIX", key, "GET", string(entry.Key)) - i, e := provider.memcacheClient.Get(string(entry.Key)) - if e == nil && i != nil { - res, err := http.ReadResponse(bufio.NewReader(bytes.NewBuffer(i.Value)), req) + i, e := provider.getFromMemcached(string(entry.Value)) + if e == nil { + res, err := http.ReadResponse(bufio.NewReader(bytes.NewBuffer(i)), req) if err == nil { rfc.ValidateETag(res, validator) if validator.Matched { @@ -257,12 +266,16 @@ func (provider *NutsMemcached) Set(key string, value []byte, url t.URL, ttl time if ttl == 0 { ttl = url.TTL.Duration } + // Only for memcached (to overcome 250 bytes key limit) + //memcachedKey := uuid.New().String() + memcachedKey := key // set to nuts (normal TTL) { err := provider.DB.Update(func(tx *nutsdb.Tx) error { - // No value is stored, value is stored in memcached - return tx.Put(bucket, []byte(key), []byte{}, uint32(ttl.Seconds())) + + // key: cache-key, value: memcached-key + return tx.Put(bucket, []byte(key), []byte(memcachedKey), uint32(ttl.Seconds())) }) if err != nil { @@ -275,8 +288,8 @@ func (provider *NutsMemcached) Set(key string, value []byte, url t.URL, ttl time staleTtl := int32((provider.stale + ttl).Seconds()) { err := provider.DB.Update(func(tx *nutsdb.Tx) error { - // No value is stored, value is stored in memcached - return tx.Put(bucket, []byte(StalePrefix+key), []byte{}, uint32(staleTtl)) + // key: "STALE_" + cache-key, value: memcached-key + return tx.Put(bucket, []byte(StalePrefix+key), []byte(memcachedKey), uint32(staleTtl)) }) if err != nil { @@ -285,38 +298,36 @@ func (provider *NutsMemcached) Set(key string, value []byte, url t.URL, ttl time } // set to memcached with stale TTL - { - // Reminder: the key must be at most 250 bytes in length - //fmt.Println("memcached SET", key) - err := provider.memcacheClient.Set( - &memcache.Item{ - Key: key, - Value: value, - Expiration: staleTtl, - }, - ) - if err != nil { - provider.logger.Sugar().Errorf("Impossible to set value into Memcached, %v", err) - } - } + _ = provider.setToMemcached(memcachedKey, value, staleTtl) return nil } // Delete method will delete the response in Nuts provider if exists corresponding to key param func (provider *NutsMemcached) Delete(key string) { + memcachedKey, _ := provider.getFromNuts(key) + + // delete from memcached + if memcachedKey != "" { + _ = provider.delFromMemcached(memcachedKey) + } + + // delete from nuts _ = provider.DB.Update(func(tx *nutsdb.Tx) error { return tx.Delete(bucket, []byte(key)) }) } // DeleteMany method will delete the responses in Nuts provider if exists corresponding to the regex key param -func (provider *NutsMemcached) DeleteMany(key string) { +func (provider *NutsMemcached) DeleteMany(keyReg string) { _ = provider.DB.Update(func(tx *nutsdb.Tx) error { - if entries, err := tx.PrefixSearchScan(bucket, []byte(""), key, 0, nutsLimit); err != nil { + if entries, err := tx.PrefixSearchScan(bucket, []byte(""), keyReg, 0, nutsLimit); err != nil { return err } else { for _, entry := range entries { + // delete from memcached + _ = provider.delFromMemcached(string(entry.Value)) + // delete from nuts _ = tx.Delete(bucket, entry.Key) } } @@ -335,3 +346,58 @@ func (provider *NutsMemcached) Reset() error { return tx.DeleteBucket(1, bucket) }) } + +func (provider *NutsMemcached) getFromNuts(nutsKey string) (memcachedKey string, err error) { + err = provider.DB.View(func(tx *nutsdb.Tx) error { + i, e := tx.Get(bucket, []byte(nutsKey)) + if i != nil { + memcachedKey = string(i.Value) + } + return e + }) + return +} + +// Reminder: the memcachedKey must be at most 250 bytes in length +func (provider *NutsMemcached) setToMemcached(memcachedKey string, value []byte, ttl int32) (err error) { + //fmt.Println("memcached SET", key) + // err = provider.memcacheClient.Set( + // &memcache.Item{ + // Key: memcachedKey, + // Value: value, + // Expiration: ttl, + // }, + // ) + //if err != nil { + // provider.logger.Sugar().Errorf("Failed to set into memcached, %v", err) + // } + ok := provider.ristrettoCache.Set(memcachedKey, value, int64(len(value))) + if !ok { + provider.logger.Sugar().Debugf("Value not set to cache, key=%v", memcachedKey) + } + return +} + +// Reminder: the memcachedKey must be at most 250 bytes in length +func (provider *NutsMemcached) getFromMemcached(memcachedKey string) (value []byte, err error) { + //fmt.Println("memcached GET", key) + // i, err := provider.memcacheClient.Get(memcachedKey) + // if err == nil && i != nil { + // value = i.Value + // } else { + // provider.logger.Sugar().Errorf("Failed to get from memcached, %v", err) + // } + rawValue, found := provider.ristrettoCache.Get(memcachedKey) + if !found { + provider.logger.Sugar().Debugf("Failed to get from cache, key=%v", memcachedKey) + return nil, errors.New("failed to get from cache") + } + value = rawValue.([]byte) + return +} + +func (provider *NutsMemcached) delFromMemcached(memcachedKey string) (err error) { + //err = provider.memcacheClient.Delete(memcachedKey) + provider.ristrettoCache.Del(memcachedKey) + return +} From 92c02be342dae26c3872b50a83e3f7770f92b489 Mon Sep 17 00:00:00 2001 From: Vincent Jordan Date: Mon, 29 Apr 2024 13:51:15 +0200 Subject: [PATCH 4/6] feat(storers): Add get/set multilevel support and rebase --- pkg/api/souin.go | 13 +- pkg/storage/nutsMemcachedProvider.go | 257 ++++++++++++++++----------- plugins/caddy/httpcache.go | 6 + 3 files changed, 169 insertions(+), 107 deletions(-) diff --git a/pkg/api/souin.go b/pkg/api/souin.go index 4d415a4d3..0e430b6de 100644 --- a/pkg/api/souin.go +++ b/pkg/api/souin.go @@ -135,12 +135,13 @@ func (s *SouinAPI) listKeys(search string) []string { } var storageToInfiniteTTLMap = map[string]time.Duration{ - "BADGER": 365 * 24 * time.Hour, - "ETCD": 365 * 24 * time.Hour, - "NUTS": 0, - "OLRIC": 365 * 24 * time.Hour, - "OTTER": 365 * 24 * time.Hour, - "REDIS": 0, + "BADGER": 365 * 24 * time.Hour, + "ETCD": 365 * 24 * time.Hour, + "NUTS": 0, + "NUTS_MEMCACHED": 0, + "OLRIC": 365 * 24 * time.Hour, + "OTTER": 365 * 24 * time.Hour, + "REDIS": 0, } func (s *SouinAPI) purgeMapping() { diff --git a/pkg/storage/nutsMemcachedProvider.go b/pkg/storage/nutsMemcachedProvider.go index 0752adb90..e7e893966 100644 --- a/pkg/storage/nutsMemcachedProvider.go +++ b/pkg/storage/nutsMemcachedProvider.go @@ -1,7 +1,6 @@ package storage import ( - "bufio" "bytes" "encoding/json" "errors" @@ -10,17 +9,31 @@ import ( "strings" "time" + "github.com/darkweak/souin/configurationtypes" t "github.com/darkweak/souin/configurationtypes" "github.com/darkweak/souin/pkg/rfc" "github.com/darkweak/souin/pkg/storage/types" "github.com/dgraph-io/ristretto" "github.com/imdario/mergo" "github.com/nutsdb/nutsdb" + lz4 "github.com/pierrec/lz4/v4" "go.uber.org/zap" ) var nutsMemcachedInstanceMap = map[string]*nutsdb.DB{} +// Why NutsMemcached? +// --- +// The NutsMemcached storage backend is composed of two different storage backends: +// 1. NutsDB: for the cache key index (i.e., IDX_ keys). +// 2. Memcached: for the cache content. +// There are two storage backends because: +// 1. is a "non forgetting" storage backend (NutsDB, for the index). Keys will be kept until their TTL expires. +// → if it was handled by a storage backend that can preemptively evict, you might evict IDX_ keys, which you wouldn't want. +// You need to make sure index and content stays in sync. +// 2. is "forgetting" storage backend (Memcached, for the data). Cache data will be pre-emptively evicted (i.e., before TTL is reached). +// → it makes it possible to put limits on total RAM/disk usage. + // NutsMemcached provider type type NutsMemcached struct { *nutsdb.DB @@ -30,51 +43,52 @@ type NutsMemcached struct { ristrettoCache *ristretto.Cache } -// const ( -// bucket = "souin-bucket" -// nutsLimit = 1 << 16 -// ) - -// func sanitizeProperties(m map[string]interface{}) map[string]interface{} { -// iotas := []string{"RWMode", "StartFileLoadingMode"} -// for _, i := range iotas { -// if v := m[i]; v != nil { -// currentMode := nutsdb.FileIO -// switch v { -// case 1: -// currentMode = nutsdb.MMap -// } -// m[i] = currentMode -// } -// } - -// for _, i := range []string{"SegmentSize", "NodeNum", "MaxFdNumsInCache"} { -// if v := m[i]; v != nil { -// m[i], _ = v.(int64) -// } -// } - -// if v := m["EntryIdxMode"]; v != nil { -// m["EntryIdxMode"] = nutsdb.HintKeyValAndRAMIdxMode -// switch v { -// case 1: -// m["EntryIdxMode"] = nutsdb.HintKeyAndRAMIdxMode -// } -// } - -// if v := m["SyncEnable"]; v != nil { -// m["SyncEnable"] = true -// if b, ok := v.(bool); ok { -// m["SyncEnable"] = b -// } else if s, ok := v.(string); ok { -// m["SyncEnable"], _ = strconv.ParseBool(s) -// } -// } - -// return m -// } - -// NutsConnectionFactory function create new Nuts instance +// Below is already defined in the original Nuts provider. +/* const ( + bucket = "souin-bucket" + nutsLimit = 1 << 16 +) + +func sanitizeProperties(m map[string]interface{}) map[string]interface{} { + iotas := []string{"RWMode", "StartFileLoadingMode"} + for _, i := range iotas { + if v := m[i]; v != nil { + currentMode := nutsdb.FileIO + switch v { + case 1: + currentMode = nutsdb.MMap + } + m[i] = currentMode + } + } + + for _, i := range []string{"SegmentSize", "NodeNum", "MaxFdNumsInCache"} { + if v := m[i]; v != nil { + m[i], _ = v.(int64) + } + } + + if v := m["EntryIdxMode"]; v != nil { + m["EntryIdxMode"] = nutsdb.HintKeyValAndRAMIdxMode + switch v { + case 1: + m["EntryIdxMode"] = nutsdb.HintKeyAndRAMIdxMode + } + } + + if v := m["SyncEnable"]; v != nil { + m["SyncEnable"] = true + if b, ok := v.(bool); ok { + m["SyncEnable"] = b + } else if s, ok := v.(string); ok { + m["SyncEnable"], _ = strconv.ParseBool(s) + } + } + + return m +} */ + +// NutsConnectionFactory function create new NutsMemcached instance func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.Storer, error) { dc := c.GetDefaultCache() nutsConfiguration := dc.GetNutsMemcached() @@ -88,6 +102,10 @@ func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.S // Use: github.com/nutsdb/nutsdb v0.14.0 //nutsOptions.EntryIdxMode = nutsdb.HintBPTSparseIdxMode + // EntryIdxMode will affect the size of the key index in memory. + // → since this storage backend has no limit on memory usage, it has to be chosen depending on + // the max number of cache keys that will be kept in flight. + if nutsConfiguration.Configuration != nil { var parsedNuts nutsdb.Options nutsConfiguration.Configuration = sanitizeProperties(nutsConfiguration.Configuration.(map[string]interface{})) @@ -122,6 +140,7 @@ func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.S return nil, e } + // Ristretto config var numCounters int64 = 1e7 // number of keys to track frequency of (10M). var maxCost int64 = 1 << 30 // maximum cost of cache (1GB). if nutsConfiguration.Configuration != nil { @@ -142,7 +161,8 @@ func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.S BufferItems: 64, // number of keys per Get buffer. }) if err != nil { - panic(err) + c.GetLogger().Sugar().Error("Impossible to make new Ristretto cache.", err) + return nil, e } instance := &NutsMemcached{ @@ -167,10 +187,13 @@ func (provider *NutsMemcached) ListKeys() []string { keys := []string{} e := provider.DB.View(func(tx *nutsdb.Tx) error { - e, _ := tx.GetAll(bucket) + e, _ := tx.PrefixScan(bucket, []byte(MappingKeyPrefix), 0, 100) for _, k := range e { - if !strings.Contains(string(k.Key), surrogatePrefix) { - keys = append(keys, string(k.Key)) + mapping, err := decodeMapping(k.Value) + if err == nil { + for _, v := range mapping.Mapping { + keys = append(keys, v.RealKey) + } } } return nil @@ -217,8 +240,8 @@ func (provider *NutsMemcached) Get(key string) (item []byte) { } // Prefix method returns the populated response if exists, empty response then -func (provider *NutsMemcached) Prefix(key string, req *http.Request, validator *rfc.Revalidator) *http.Response { - var result *http.Response +func (provider *NutsMemcached) Prefix(key string) []string { + result := []string{} _ = provider.DB.View(func(tx *nutsdb.Tx) error { prefix := []byte(key) @@ -227,32 +250,7 @@ func (provider *NutsMemcached) Prefix(key string, req *http.Request, validator * return err } else { for _, entry := range entries { - if varyVoter(key, req, string(entry.Key)) { - // TODO: improve this - // Store only response header in nuts and avoid query to memcached on each vary - // E.g, rfc.ValidateETag on NutsDB header value, retrieve response body later from memcached. - - // Reminder: the key must be at most 250 bytes in length - //fmt.Println("memcached PREFIX", key, "GET", string(entry.Key)) - i, e := provider.getFromMemcached(string(entry.Value)) - if e == nil { - res, err := http.ReadResponse(bufio.NewReader(bytes.NewBuffer(i)), req) - if err == nil { - rfc.ValidateETag(res, validator) - if validator.Matched { - provider.logger.Sugar().Debugf("The stored key %s matched the current iteration key ETag %+v", string(entry.Key), validator) - result = res - return nil - } - - provider.logger.Sugar().Debugf("The stored key %s didn't match the current iteration key ETag %+v", string(entry.Key), validator) - } else { - provider.logger.Sugar().Errorf("An error occured while reading response for the key %s: %v", string(entry.Key), err) - } - } else { - provider.logger.Sugar().Errorf("An error occured while reading memcached for the key %s: %v", string(entry.Key), err) - } - } + result = append(result, string(entry.Key)) } } return nil @@ -261,45 +259,102 @@ func (provider *NutsMemcached) Prefix(key string, req *http.Request, validator * return result } -// Set method will store the response in Nuts provider -func (provider *NutsMemcached) Set(key string, value []byte, url t.URL, ttl time.Duration) error { - if ttl == 0 { - ttl = url.TTL.Duration - } - // Only for memcached (to overcome 250 bytes key limit) - //memcachedKey := uuid.New().String() - memcachedKey := key +// GetMultiLevel tries to load the key and check if one of linked keys is a fresh/stale candidate. +func (provider *NutsMemcached) GetMultiLevel(key string, req *http.Request, validator *rfc.Revalidator) (fresh *http.Response, stale *http.Response) { + _ = provider.DB.View(func(tx *nutsdb.Tx) error { + i, e := tx.Get(bucket, []byte(MappingKeyPrefix+key)) + if e != nil && !errors.Is(e, nutsdb.ErrKeyNotFound) { + return e + } - // set to nuts (normal TTL) - { - err := provider.DB.Update(func(tx *nutsdb.Tx) error { + var val []byte + if i != nil { + val = i.Value + } + fresh, stale, e = mappingElection(provider, val, req, validator, provider.logger) - // key: cache-key, value: memcached-key - return tx.Put(bucket, []byte(key), []byte(memcachedKey), uint32(ttl.Seconds())) - }) + return e + }) + return +} + +// SetMultiLevel tries to store the key with the given value and update the mapping key to store metadata. +func (provider *NutsMemcached) SetMultiLevel(baseKey, variedKey string, value []byte, variedHeaders http.Header, etag string, duration time.Duration, realKey string) error { + now := time.Now() + + compressed := new(bytes.Buffer) + if _, err := lz4.NewWriter(compressed).ReadFrom(bytes.NewReader(value)); err != nil { + provider.logger.Sugar().Errorf("Impossible to compress the key %s into Nuts, %v", variedKey, err) + return err + } + { + // matchedURL is only use when ttl == 0 + ttl := duration + provider.stale + url := t.URL{ + TTL: configurationtypes.Duration{Duration: ttl}, + } + err := provider.Set(variedKey, compressed.Bytes(), url, ttl) if err != nil { - provider.logger.Sugar().Errorf("Impossible to set value into Nuts, %v", err) return err } } - // set to nuts (stale TTL) - staleTtl := int32((provider.stale + ttl).Seconds()) + err := provider.DB.Update(func(tx *nutsdb.Tx) error { + mappingKey := MappingKeyPrefix + baseKey + item, e := tx.Get(bucket, []byte(mappingKey)) + if e != nil && !errors.Is(e, nutsdb.ErrKeyNotFound) { + provider.logger.Sugar().Errorf("Impossible to get the base key %s in Nuts, %v", baseKey, e) + return e + } + + var val []byte + if item != nil { + val = item.Value + } + + val, e = mappingUpdater(variedKey, val, provider.logger, now, now.Add(duration), now.Add(duration+provider.stale), variedHeaders, etag, realKey) + if e != nil { + return e + } + + provider.logger.Sugar().Debugf("Store the new mapping for the key %s in Nuts", variedKey) + + return tx.Put(bucket, []byte(mappingKey), val, nutsdb.Persistent) + }) + + if err != nil { + provider.logger.Sugar().Errorf("Impossible to set value into Nuts, %v", err) + } + + return err +} + +// Set method will store the response in Nuts provider +func (provider *NutsMemcached) Set(key string, value []byte, url t.URL, duration time.Duration) error { + if duration == 0 { + duration = url.TTL.Duration + } + // Only for memcached (to overcome 250 bytes key limit) + //memcachedKey := uuid.New().String() + // Disabled for ristretto to improve performances + memcachedKey := key + + // set to nuts { err := provider.DB.Update(func(tx *nutsdb.Tx) error { - // key: "STALE_" + cache-key, value: memcached-key - return tx.Put(bucket, []byte(StalePrefix+key), []byte(memcachedKey), uint32(staleTtl)) + // key: cache-key, value: memcached-key + return tx.Put(bucket, []byte(key), []byte(memcachedKey), uint32(duration.Seconds())) }) if err != nil { provider.logger.Sugar().Errorf("Impossible to set value into Nuts, %v", err) + return err } } - // set to memcached with stale TTL - _ = provider.setToMemcached(memcachedKey, value, staleTtl) - + // set to memcached + _ = provider.setToMemcached(memcachedKey, value, int32(duration.Seconds())) return nil } @@ -373,7 +428,7 @@ func (provider *NutsMemcached) setToMemcached(memcachedKey string, value []byte, // } ok := provider.ristrettoCache.Set(memcachedKey, value, int64(len(value))) if !ok { - provider.logger.Sugar().Debugf("Value not set to cache, key=%v", memcachedKey) + provider.logger.Sugar().Debugf("Value not set to ristretto cache, key=%v", memcachedKey) } return } diff --git a/plugins/caddy/httpcache.go b/plugins/caddy/httpcache.go index 498f66a66..7152d8718 100644 --- a/plugins/caddy/httpcache.go +++ b/plugins/caddy/httpcache.go @@ -54,6 +54,8 @@ type SouinCaddyMiddleware struct { Nuts configurationtypes.CacheProvider `json:"nuts,omitempty"` // Configure the Otter cache storage. Otter configurationtypes.CacheProvider `json:"otter,omitempty"` + // Configure the Otter cache storage. + NutsMemcached configurationtypes.CacheProvider `json:"nuts_memcached,omitempty"` // Enable the Etcd distributed cache storage. Etcd configurationtypes.CacheProvider `json:"etcd,omitempty"` // Enable the Redis distributed cache storage. @@ -95,6 +97,7 @@ func (s *SouinCaddyMiddleware) configurationPropertyMapper() error { Badger: s.Badger, Nuts: s.Nuts, Otter: s.Otter, + NutsMemcached: s.NutsMemcached, Key: s.Key, DefaultCacheControl: s.DefaultCacheControl, CacheName: s.CacheName, @@ -222,6 +225,9 @@ func (s *SouinCaddyMiddleware) FromApp(app *SouinApp) error { if dc.Otter.Path == "" && dc.Otter.Configuration == nil { s.Configuration.DefaultCache.Otter = appDc.Otter } + if dc.NutsMemcached.Path == "" && dc.NutsMemcached.Configuration == nil { + s.Configuration.DefaultCache.NutsMemcached = appDc.NutsMemcached + } if dc.Regex.Exclude == "" { s.Configuration.DefaultCache.Regex.Exclude = appDc.Regex.Exclude } From 5e72cd4e7a9d471e277d06b47b07bac90329df2e Mon Sep 17 00:00:00 2001 From: Vincent Jordan Date: Tue, 30 Apr 2024 10:54:33 +0200 Subject: [PATCH 5/6] feat(storers): Add NutsMemcached storage tests and fix NutsDB multi-instance reuse. --- pkg/storage/nutsMemcachedProvider.go | 49 ++++++---- pkg/storage/nutsMemcachedProvider_test.go | 110 ++++++++++++++++++++++ 2 files changed, 140 insertions(+), 19 deletions(-) create mode 100644 pkg/storage/nutsMemcachedProvider_test.go diff --git a/pkg/storage/nutsMemcachedProvider.go b/pkg/storage/nutsMemcachedProvider.go index e7e893966..813f41156 100644 --- a/pkg/storage/nutsMemcachedProvider.go +++ b/pkg/storage/nutsMemcachedProvider.go @@ -125,21 +125,6 @@ func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.S } } - if instance, ok := nutsMemcachedInstanceMap[nutsOptions.Dir]; ok && instance != nil { - return &NutsMemcached{ - DB: instance, - stale: dc.GetStale(), - logger: c.GetLogger(), - }, nil - } - - db, e := nutsdb.Open(nutsOptions) - - if e != nil { - c.GetLogger().Sugar().Error("Impossible to open the Nuts DB.", e) - return nil, e - } - // Ristretto config var numCounters int64 = 1e7 // number of keys to track frequency of (10M). var maxCost int64 = 1 << 30 // maximum cost of cache (1GB). @@ -162,6 +147,25 @@ func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.S }) if err != nil { c.GetLogger().Sugar().Error("Impossible to make new Ristretto cache.", err) + return nil, err + } + //c.GetLogger().Sugar().Debugf("Ristretto cache was created with NumCounters=%d, MaxCost=%d", numCounters, maxCost) + + // If multiple caches are created on the same directory, reuse the same NutsDB instance. + // E.g., in automated tests. + if instance, ok := nutsMemcachedInstanceMap[nutsOptions.Dir]; ok && instance != nil { + return &NutsMemcached{ + DB: instance, + stale: dc.GetStale(), + logger: c.GetLogger(), + //memcacheClient: memcache.New("127.0.0.1:11211"), // hardcoded for now + ristrettoCache: ristrettoCache, + }, nil + } + + db, e := nutsdb.Open(nutsOptions) + if e != nil { + c.GetLogger().Sugar().Error("Impossible to open the Nuts DB.", e) return nil, e } @@ -354,7 +358,7 @@ func (provider *NutsMemcached) Set(key string, value []byte, url t.URL, duration } // set to memcached - _ = provider.setToMemcached(memcachedKey, value, int32(duration.Seconds())) + _ = provider.setToMemcached(memcachedKey, value, duration) return nil } @@ -414,7 +418,7 @@ func (provider *NutsMemcached) getFromNuts(nutsKey string) (memcachedKey string, } // Reminder: the memcachedKey must be at most 250 bytes in length -func (provider *NutsMemcached) setToMemcached(memcachedKey string, value []byte, ttl int32) (err error) { +func (provider *NutsMemcached) setToMemcached(memcachedKey string, value []byte, ttl time.Duration) (err error) { //fmt.Println("memcached SET", key) // err = provider.memcacheClient.Set( // &memcache.Item{ @@ -426,9 +430,16 @@ func (provider *NutsMemcached) setToMemcached(memcachedKey string, value []byte, //if err != nil { // provider.logger.Sugar().Errorf("Failed to set into memcached, %v", err) // } - ok := provider.ristrettoCache.Set(memcachedKey, value, int64(len(value))) + + ok := provider.ristrettoCache.SetWithTTL(memcachedKey, value, int64(len(value)), ttl) if !ok { - provider.logger.Sugar().Debugf("Value not set to ristretto cache, key=%v", memcachedKey) + provider.logger.Sugar().Debugf( + "Value not set to ristretto cache, key=%v ttl=%.2fs len=%d", + memcachedKey, ttl.Seconds(), len(value), + ) + // Note: failed to store is not considered an error because Ristretto doesn't guarantee + // a value is set or not. + // See https://pkg.go.dev/github.com/dgraph-io/ristretto@v0.1.1#Cache.Set } return } diff --git a/pkg/storage/nutsMemcachedProvider_test.go b/pkg/storage/nutsMemcachedProvider_test.go new file mode 100644 index 000000000..d18d10840 --- /dev/null +++ b/pkg/storage/nutsMemcachedProvider_test.go @@ -0,0 +1,110 @@ +package storage + +import ( + "fmt" + "testing" + + "github.com/darkweak/souin/pkg/storage/types" + "github.com/darkweak/souin/tests" + + "time" + + "github.com/darkweak/souin/configurationtypes" + "github.com/darkweak/souin/errors" +) + +func getNutsMemcachedClientAndMatchedURL(key string) (types.Storer, configurationtypes.URL) { + return GetCacheProviderClientAndMatchedURL( + key, + func() configurationtypes.AbstractConfigurationInterface { + return tests.MockConfiguration(tests.NutsConfiguration) + }, + func(config configurationtypes.AbstractConfigurationInterface) (types.Storer, error) { + provider, _ := NutsMemcachedConnectionFactory(config) + _ = provider.Init() + + return provider, nil + }, + ) +} + +func TestNutsMemcached_ConnectionFactory(t *testing.T) { + c := tests.MockConfiguration(tests.NutsConfiguration) + r, err := NutsMemcachedConnectionFactory(c) + + if nil != err { + errors.GenerateError(t, "Shouldn't have panic") + } + + if nil == r { + errors.GenerateError(t, "Nuts should be instanciated") + } + + if nil == r.(*NutsMemcached).DB { + errors.GenerateError(t, "Nuts database should be accesible") + } +} + +func TestNutsMemcached_IShouldBeAbleToReadAndWriteData(t *testing.T) { + client, matchedURL := getNutsMemcachedClientAndMatchedURL("Test") + + _ = client.Set("Test", []byte(BASE_VALUE), matchedURL, time.Duration(20)*time.Second) + time.Sleep(1 * time.Second) + + res := client.Get("Test") + if res == nil || len(res) <= 0 { + errors.GenerateError(t, fmt.Sprintf("Key '%s' should exist", BASE_VALUE)) + } + if BASE_VALUE != string(res) { + errors.GenerateError(t, fmt.Sprintf("'%s' not corresponding to '%s'", string(res), BASE_VALUE)) + } +} + +func TestNutsMemcached_GetRequestInCache(t *testing.T) { + c := tests.MockConfiguration(tests.NutsConfiguration) + client, _ := NutsMemcachedConnectionFactory(c) + res := client.Get(NONEXISTENTKEY) + if 0 < len(res) { + errors.GenerateError(t, fmt.Sprintf("Key %s should not exist", NONEXISTENTKEY)) + } +} + +func TestNutsMemcached_GetSetRequestInCache_OneByte(t *testing.T) { + client, matchedURL := getNutsMemcachedClientAndMatchedURL(BYTEKEY) + _ = client.Set(BYTEKEY, []byte("A"), matchedURL, time.Duration(20)*time.Second) + time.Sleep(1 * time.Second) + + res := client.Get(BYTEKEY) + if len(res) == 0 { + errors.GenerateError(t, fmt.Sprintf("Key %s should exist", BYTEKEY)) + } + + if string(res) != "A" { + errors.GenerateError(t, fmt.Sprintf("%s not corresponding to %v", res, 65)) + } +} + +func TestNutsMemcached_SetRequestInCache_TTL(t *testing.T) { + key := "MyEmptyKey" + client, matchedURL := getNutsMemcachedClientAndMatchedURL(key) + nv := []byte("Hello world") + setValueThenVerify(client, key, nv, matchedURL, time.Duration(20)*time.Second, t) +} + +func TestNutsMemcached_DeleteRequestInCache(t *testing.T) { + client, _ := NutsMemcachedConnectionFactory(tests.MockConfiguration(tests.NutsConfiguration)) + client.Delete(BYTEKEY) + time.Sleep(1 * time.Second) + if 0 < len(client.Get(BYTEKEY)) { + errors.GenerateError(t, fmt.Sprintf("Key %s should not exist", BYTEKEY)) + } +} + +func TestNutsMemcached_Init(t *testing.T) { + client, _ := NutsMemcachedConnectionFactory(tests.MockConfiguration(tests.NutsConfiguration)) + err := client.Init() + + if nil != err { + errors.GenerateError(t, "Impossible to init NutsMemcached provider") + } +} From f380cc3a4993caf3e12c96af34a79a6345e106f8 Mon Sep 17 00:00:00 2001 From: Vincent Jordan Date: Tue, 30 Apr 2024 15:27:19 +0200 Subject: [PATCH 6/6] feat(storers): Change NutsDB bucket name --- pkg/storage/nutsMemcachedProvider.go | 44 +++++++++++++++------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/pkg/storage/nutsMemcachedProvider.go b/pkg/storage/nutsMemcachedProvider.go index 813f41156..470776c46 100644 --- a/pkg/storage/nutsMemcachedProvider.go +++ b/pkg/storage/nutsMemcachedProvider.go @@ -37,8 +37,9 @@ var nutsMemcachedInstanceMap = map[string]*nutsdb.DB{} // NutsMemcached provider type type NutsMemcached struct { *nutsdb.DB - stale time.Duration - logger *zap.Logger + bucketName string + stale time.Duration + logger *zap.Logger //memcacheClient *memcache.Client ristrettoCache *ristretto.Cache } @@ -91,6 +92,7 @@ func sanitizeProperties(m map[string]interface{}) map[string]interface{} { // NutsConnectionFactory function create new NutsMemcached instance func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.Storer, error) { dc := c.GetDefaultCache() + cacheName := dc.GetCacheName() nutsConfiguration := dc.GetNutsMemcached() nutsOptions := nutsdb.DefaultOptions nutsOptions.Dir = "/tmp/souin-nuts-memcached" @@ -155,9 +157,10 @@ func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.S // E.g., in automated tests. if instance, ok := nutsMemcachedInstanceMap[nutsOptions.Dir]; ok && instance != nil { return &NutsMemcached{ - DB: instance, - stale: dc.GetStale(), - logger: c.GetLogger(), + DB: instance, + bucketName: cacheName, + stale: dc.GetStale(), + logger: c.GetLogger(), //memcacheClient: memcache.New("127.0.0.1:11211"), // hardcoded for now ristrettoCache: ristrettoCache, }, nil @@ -170,9 +173,10 @@ func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.S } instance := &NutsMemcached{ - DB: db, - stale: dc.GetStale(), - logger: c.GetLogger(), + DB: db, + bucketName: cacheName, + stale: dc.GetStale(), + logger: c.GetLogger(), //memcacheClient: memcache.New("127.0.0.1:11211"), // hardcoded for now ristrettoCache: ristrettoCache, } @@ -191,7 +195,7 @@ func (provider *NutsMemcached) ListKeys() []string { keys := []string{} e := provider.DB.View(func(tx *nutsdb.Tx) error { - e, _ := tx.PrefixScan(bucket, []byte(MappingKeyPrefix), 0, 100) + e, _ := tx.PrefixScan(provider.bucketName, []byte(MappingKeyPrefix), 0, 100) for _, k := range e { mapping, err := decodeMapping(k.Value) if err == nil { @@ -215,7 +219,7 @@ func (provider *NutsMemcached) MapKeys(prefix string) map[string]string { keys := map[string]string{} e := provider.DB.View(func(tx *nutsdb.Tx) error { - e, _ := tx.GetAll(bucket) + e, _ := tx.GetAll(provider.bucketName) for _, k := range e { if strings.HasPrefix(string(k.Key), prefix) { nk, _ := strings.CutPrefix(string(k.Key), prefix) @@ -250,7 +254,7 @@ func (provider *NutsMemcached) Prefix(key string) []string { _ = provider.DB.View(func(tx *nutsdb.Tx) error { prefix := []byte(key) - if entries, err := tx.PrefixSearchScan(bucket, prefix, "^({|$)", 0, 50); err != nil { + if entries, err := tx.PrefixSearchScan(provider.bucketName, prefix, "^({|$)", 0, 50); err != nil { return err } else { for _, entry := range entries { @@ -266,7 +270,7 @@ func (provider *NutsMemcached) Prefix(key string) []string { // GetMultiLevel tries to load the key and check if one of linked keys is a fresh/stale candidate. func (provider *NutsMemcached) GetMultiLevel(key string, req *http.Request, validator *rfc.Revalidator) (fresh *http.Response, stale *http.Response) { _ = provider.DB.View(func(tx *nutsdb.Tx) error { - i, e := tx.Get(bucket, []byte(MappingKeyPrefix+key)) + i, e := tx.Get(provider.bucketName, []byte(MappingKeyPrefix+key)) if e != nil && !errors.Is(e, nutsdb.ErrKeyNotFound) { return e } @@ -306,7 +310,7 @@ func (provider *NutsMemcached) SetMultiLevel(baseKey, variedKey string, value [] err := provider.DB.Update(func(tx *nutsdb.Tx) error { mappingKey := MappingKeyPrefix + baseKey - item, e := tx.Get(bucket, []byte(mappingKey)) + item, e := tx.Get(provider.bucketName, []byte(mappingKey)) if e != nil && !errors.Is(e, nutsdb.ErrKeyNotFound) { provider.logger.Sugar().Errorf("Impossible to get the base key %s in Nuts, %v", baseKey, e) return e @@ -324,7 +328,7 @@ func (provider *NutsMemcached) SetMultiLevel(baseKey, variedKey string, value [] provider.logger.Sugar().Debugf("Store the new mapping for the key %s in Nuts", variedKey) - return tx.Put(bucket, []byte(mappingKey), val, nutsdb.Persistent) + return tx.Put(provider.bucketName, []byte(mappingKey), val, nutsdb.Persistent) }) if err != nil { @@ -348,7 +352,7 @@ func (provider *NutsMemcached) Set(key string, value []byte, url t.URL, duration { err := provider.DB.Update(func(tx *nutsdb.Tx) error { // key: cache-key, value: memcached-key - return tx.Put(bucket, []byte(key), []byte(memcachedKey), uint32(duration.Seconds())) + return tx.Put(provider.bucketName, []byte(key), []byte(memcachedKey), uint32(duration.Seconds())) }) if err != nil { @@ -373,21 +377,21 @@ func (provider *NutsMemcached) Delete(key string) { // delete from nuts _ = provider.DB.Update(func(tx *nutsdb.Tx) error { - return tx.Delete(bucket, []byte(key)) + return tx.Delete(provider.bucketName, []byte(key)) }) } // DeleteMany method will delete the responses in Nuts provider if exists corresponding to the regex key param func (provider *NutsMemcached) DeleteMany(keyReg string) { _ = provider.DB.Update(func(tx *nutsdb.Tx) error { - if entries, err := tx.PrefixSearchScan(bucket, []byte(""), keyReg, 0, nutsLimit); err != nil { + if entries, err := tx.PrefixSearchScan(provider.bucketName, []byte(""), keyReg, 0, nutsLimit); err != nil { return err } else { for _, entry := range entries { // delete from memcached _ = provider.delFromMemcached(string(entry.Value)) // delete from nuts - _ = tx.Delete(bucket, entry.Key) + _ = tx.Delete(provider.bucketName, entry.Key) } } return nil @@ -402,13 +406,13 @@ func (provider *NutsMemcached) Init() error { // Reset method will reset or close provider func (provider *NutsMemcached) Reset() error { return provider.DB.Update(func(tx *nutsdb.Tx) error { - return tx.DeleteBucket(1, bucket) + return tx.DeleteBucket(1, provider.bucketName) }) } func (provider *NutsMemcached) getFromNuts(nutsKey string) (memcachedKey string, err error) { err = provider.DB.View(func(tx *nutsdb.Tx) error { - i, e := tx.Get(bucket, []byte(nutsKey)) + i, e := tx.Get(provider.bucketName, []byte(nutsKey)) if i != nil { memcachedKey = string(i.Value) }