Merge pull request '[Perf] Member state' (#5) from perf/member-state into main

Reviewed-on: https://git.anhgelus.world/anhgelus/les-copaings-bot/pulls/5

4 files changed, 316 insertions, 146 deletions

diff --git a/user/level.go b/user/level.go
index e7b96af..92c10ab 100644
--- a/user/level.go
+++ b/user/level.go
@@ -1,6 +1,7 @@
 package user
 
 import (
+	"context"
 	"slices"
 	"sync"
 	"time"
@@ -45,7 +46,7 @@ func onNewLevel(s bot.Session, m *user.Member, level uint) {
 	}
 }
 
-func (c *Copaing) OnNewLevel(s *discordgo.Session, level uint) {
+func (c *CopaingCached) onNewLevel(s *discordgo.Session, level uint) {
 	m, err := s.GuildAPI().Member(c.GuildID, c.DiscordID)
 	if err != nil {
 		s.Logger().Error("getting member for new level", "error", err, "user", c.DiscordID, "guild", c.GuildID)
@@ -54,65 +55,61 @@ func (c *Copaing) OnNewLevel(s *discordgo.Session, level uint) {
 	onNewLevel(s, m, level)
 }
 
-func PeriodicReducer(s *discordgo.Session) {
-	wg := &sync.WaitGroup{}
-	var cs []*Copaing
-	if err := gokord.DB.Find(&cs).Error; err != nil {
-		s.Logger().Error("fetching all copaings", "error", err)
-		return
-	}
-	cxps := make([]*cXP, len(cs))
-	for i, c := range cs {
-		if i%25 == 24 {
-			wg.Wait() // prevents spamming the DB
-		}
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			xp, err := c.GetXP(s.Logger())
-			if err != nil {
-				s.Logger().Error("getting xp", "error", err, "copaing", c.ID, "guild", c.GuildID)
-				xp = 0
-			}
-			cxps[i] = &cXP{
-				Cxp:     xp,
-				Copaing: c,
-			}
-		}()
-	}
-	wg.Wait()
-	i := 0
+func PeriodicReducer(ctx context.Context, s *discordgo.Session) {
+	PeriodicSaver(ctx, s)
+
+	s.Logger().Debug("periodic reducer")
+
+	state := GetState(ctx)
+
+	n := 0
+	var wg sync.WaitGroup
 	for _, g := range s.GuildAPI().State.Guilds() {
-		i++
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			cfg := config.GetGuildConfig(g)
-			res := gokord.DB.
-				Model(&CopaingXP{}).
-				Where("guild_id = ? and created_at < ?", g, exp.TimeStampNDaysBefore(cfg.DaysXPRemains)).
-				Delete(&CopaingXP{})
-			if res.Error != nil {
-				s.Logger().Error("removing old xp", "error", res.Error, "guild", g)
-			}
-			s.Logger().Debug("guild cleaned", "guild", g, "rows affected", res.RowsAffected)
-		}()
+		n++
+		cfg := config.GetGuildConfig(g)
+		res := gokord.DB.
+			Model(&CopaingXP{}).
+			Where("guild_id = ? and created_at < ?", g, exp.TimeStampNDaysBefore(cfg.DaysXPRemains)).
+			Delete(&CopaingXP{})
+		if res.Error != nil {
+			s.Logger().Error("removing old xp", "error", res.Error, "guild", g)
+			continue
+		}
+		s.Logger().Debug("guild cleaned", "guild", g, "rows affected", res.RowsAffected)
+
+		wg.Go(func() {
+			syncCopaings(ctx, s, state.Copaings(g))
+		})
 	}
+
 	wg.Wait()
-	for i, c := range cxps {
+
+	s.Logger().Debug("periodic reduce finished", "guilds affected", n)
+}
+
+func syncCopaings(ctx context.Context, s *discordgo.Session, ccs []CopaingCached) {
+	for i, cc := range ccs {
 		if i%50 == 49 {
 			s.Logger().Debug("sleeping...")
 			time.Sleep(15 * time.Second) // prevents spamming the API
 		}
-		oldXp := c.GetXP()
-		xp, err := c.ToCopaing().GetXP(s.Logger())
+		oldXp := cc.XP
+		err := cc.Sync(ctx)
 		if err != nil {
-			s.Logger().Error("getting xp of copaing", "error", err, "copaing", c.ID, "guild", c.GuildID)
+			s.Logger().Error("syncing copaing", "error", err, "copaing", cc.ID, "guild", cc.GuildID)
 			continue
 		}
+		xp := cc.XP
 		if exp.Level(oldXp) != exp.Level(xp) {
-			c.OnNewLevel(s, exp.Level(xp))
+			cc.onNewLevel(s, exp.Level(xp))
 		}
 	}
-	s.Logger().Debug("periodic reduce finished", "guilds affected", i)
+}
+
+func PeriodicSaver(ctx context.Context, s bot.Session) {
+	s.Logger().Debug("saving state in DB")
+	err := saveStateInDB(ctx)
+	if err != nil {
+		panic(err)
+	}
 }
diff --git a/user/member.go b/user/member.go
index 9068a6f..4969f8a 100644
--- a/user/member.go
+++ b/user/member.go
@@ -1,6 +1,7 @@
 package user
 
 import (
+	"context"
 	"time"
 
 	"github.com/anhgelus/gokord"
@@ -22,24 +23,33 @@ type CopaingXP struct {
 }
 
 type CopaingAccess interface {
-	ToCopaing() *Copaing
+	Copaing() *Copaing
 	GetXP() uint
 }
 
-func GetCopaing(discordID string, guildID string) *Copaing {
-	c := Copaing{DiscordID: discordID, GuildID: guildID}
-	if err := c.Load(); err != nil {
-		panic(err)
+func GetCopaing(ctx context.Context, discordID string, guildID string) *CopaingCached {
+	state := GetState(ctx)
+	cc, err := state.Copaing(guildID, discordID)
+	if err != nil {
+		c := Copaing{DiscordID: discordID, GuildID: guildID}
+		if err := c.load(); err != nil {
+			panic(err)
+		}
+		cc = FromCopaing(&c)
 	}
-	return &c
+	return cc
 }
 
-func (c *Copaing) Load() error {
-	return gokord.DB.
+func (c *Copaing) load() error {
+	err := gokord.DB.
 		Where("discord_id = ? and guild_id = ?", c.DiscordID, c.GuildID).
 		Preload("CopaingXPs").
 		FirstOrCreate(c).
 		Error
+	if err != nil {
+		return err
+	}
+	return err
 }
 
 func (c *Copaing) Save() error {
@@ -47,5 +57,12 @@ func (c *Copaing) Save() error {
 }
 
 func (c *Copaing) Delete() error {
+	err := gokord.DB.
+		Where("copaing_id = ? and guild_id = ?", c.ID, c.GuildID).
+		Delete(&CopaingXP{}).
+		Error
+	if err != nil {
+		return err
+	}
 	return gokord.DB.Where("guild_id = ? AND discord_id = ?", c.GuildID, c.DiscordID).Delete(c).Error
 }
diff --git a/user/state.go b/user/state.go
new file mode 100644
index 0000000..effdc80
--- /dev/null
+++ b/user/state.go
@@ -0,0 +1,220 @@
+package user
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/anhgelus/gokord"
+	"github.com/nyttikord/gokord/state"
+)
+
+var ErrSyncingUnsavedData = errors.New("trying to sync unsaved data")
+
+type XPCached struct {
+	XP   uint
+	Time time.Duration
+}
+
+type CopaingCached struct {
+	ID        uint
+	DiscordID string
+	GuildID   string
+	XP        uint
+	XPs       []XPCached
+	XPToAdd   uint
+}
+
+// copaing turns a CopaingCached into a Copaing.
+// This operation get the copaing synced with the database.
+// It doesn't:
+// - save the copaing in the database, use CopaingCached.SaveInDB for that;
+// - sync the copaing cached, use CopaingCached.Sync for that.
+// TL;DR: don't use this method, unless you know what are you doing.
+func (cc *CopaingCached) copaing() *Copaing {
+	c := &Copaing{DiscordID: cc.DiscordID, GuildID: cc.GuildID}
+	if err := c.load(); err != nil {
+		panic(err)
+	}
+	return c
+}
+
+func (cc *CopaingCached) Sync(ctx context.Context) error {
+	if cc.mustSave() {
+		return ErrSyncingUnsavedData
+	}
+	synced := FromCopaing(cc.copaing())
+	synced.XP += cc.XPToAdd
+	synced.XPToAdd = cc.XPToAdd
+	err := synced.Save(ctx)
+	if err != nil {
+		return err
+	}
+	*cc = *synced
+	return nil
+}
+
+func (cc *CopaingCached) Save(ctx context.Context) error {
+	state := GetState(ctx)
+
+	state.mu.Lock()
+	defer state.mu.Unlock()
+
+	return state.storage.Write(KeyCopaingCachedRaw(cc.GuildID, cc.DiscordID), *cc)
+}
+
+func (cc *CopaingCached) SaveInDB(ctx context.Context) error {
+	c := cc.copaing()
+	c.CopaingXPs = append(c.CopaingXPs, CopaingXP{CopaingID: c.ID, XP: cc.XPToAdd, GuildID: c.GuildID})
+	err := c.Save()
+	if err != nil {
+		return err
+	}
+	cc.XPToAdd = 0
+	return cc.Save(ctx)
+}
+
+func (cc *CopaingCached) Delete(ctx context.Context) error {
+	c := cc.copaing()
+	err := c.Delete()
+	if err != nil {
+		return err
+	}
+	state := GetState(ctx)
+
+	state.mu.Lock()
+	defer state.mu.Unlock()
+
+	return state.storage.Delete(KeyCopaingCached(c))
+}
+
+func (cc *CopaingCached) mustSave() bool {
+	return cc.XPToAdd > 0
+}
+
+func saveStateInDB(ctx context.Context) error {
+	state := GetState(ctx)
+
+	state.saveInDB.Lock()
+	defer state.saveInDB.Unlock()
+
+	for _, v := range state.storage {
+		if v.mustSave() {
+			err := v.SaveInDB(ctx)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func FromCopaing(c *Copaing) *CopaingCached {
+	return &CopaingCached{
+		ID:        c.ID,
+		DiscordID: c.DiscordID,
+		GuildID:   c.GuildID,
+		XP:        calcXP(c),
+		XPs:       generateXPs(c),
+		XPToAdd:   0,
+	}
+}
+
+const KeyCopaingCachedPrefix = "cc:"
+
+func KeyCopaingCached(c *Copaing) state.Key {
+	return KeyCopaingCachedRaw(c.GuildID, c.DiscordID)
+}
+
+func KeyCopaingCachedRaw(guildID, copaingID string) state.Key {
+	return KeyCopaingCachedPrefix + state.Key(guildID+":"+copaingID)
+}
+
+type State struct {
+	mu       sync.RWMutex
+	saveInDB sync.Mutex
+	storage  state.MapStorage[CopaingCached]
+}
+
+func NewState() *State {
+	state := &State{
+		storage: state.MapStorage[CopaingCached]{},
+	}
+	var cs []*Copaing
+	err := gokord.DB.Find(&cs).Error
+	if err != nil {
+		panic(err)
+	}
+	for _, v := range cs {
+		FromCopaing(v).Save(SetState(context.Background(), state))
+	}
+	return state
+}
+
+const ContextKeyState = "state"
+
+func GetState(ctx context.Context) *State {
+	return ctx.Value(ContextKeyState).(*State)
+}
+
+func SetState(ctx context.Context, state *State) context.Context {
+	return context.WithValue(ctx, ContextKeyState, state)
+}
+
+func (s *State) Copaing(guildID, copaingID string) (*CopaingCached, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	raw, err := s.storage.Get(KeyCopaingCachedRaw(guildID, copaingID))
+	if err != nil {
+		return nil, err
+	}
+	c := raw.(CopaingCached)
+	return &c, nil
+}
+
+func (s *State) Copaings(guild string) []CopaingCached {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	var ccs []CopaingCached
+	for _, cc := range s.storage {
+		if cc.GuildID == guild {
+			ccs = append(ccs, cc)
+		}
+	}
+	return ccs
+}
+
+func calcXP(c *Copaing) uint {
+	var sum uint
+	for _, entry := range c.CopaingXPs {
+		sum += entry.XP
+	}
+	return sum
+}
+
+func generateXPs(c *Copaing) []XPCached {
+	data := map[time.Duration]XPCached{}
+	sixH := 6 * time.Hour
+	for _, xp := range c.CopaingXPs {
+		// we add sixH at the end because we want it to be rounded to ceil
+		since := time.Since(xp.CreatedAt)/sixH + sixH
+		if v, ok := data[since]; ok {
+			v.XP += xp.XP
+		} else {
+			data[since] = XPCached{
+				Time: since,
+				XP:   xp.XP,
+			}
+		}
+	}
+	ccs := make([]XPCached, len(data))
+	i := 0
+	for _, v := range data {
+		ccs[i] = v
+		i++
+	}
+	return ccs
+}
diff --git a/user/xp.go b/user/xp.go
index dbca9de..c87c450 100644
--- a/user/xp.go
+++ b/user/xp.go
@@ -1,41 +1,36 @@
 package user
 
 import (
-	"log/slog"
+	"context"
 	"slices"
-	"sync"
+	"time"
 
-	"git.anhgelus.world/anhgelus/les-copaings-bot/config"
 	"git.anhgelus.world/anhgelus/les-copaings-bot/exp"
-	"github.com/anhgelus/gokord"
 	"github.com/nyttikord/gokord/bot"
 	"github.com/nyttikord/gokord/user"
 )
 
 type cXP struct {
-	Cxp uint
-	*Copaing
+	Cxp     uint
+	copaing *Copaing
 }
 
-func (c *cXP) ToCopaing() *Copaing {
-	return c.Copaing
+func (c *cXP) Copaing() *Copaing {
+	return c.copaing
 }
 
 func (c *cXP) GetXP() uint {
 	return c.Cxp
 }
 
-func (c *Copaing) AddXP(s bot.Session, m *user.Member, xp uint, fn func(uint, uint)) {
-	old, err := c.GetXP(s.Logger())
-	if err != nil {
-		s.Logger().Error("getting xp", "error", err, "user", m.DisplayName(), "guild", c.GuildID)
-		return
-	}
+func (cc *CopaingCached) AddXP(ctx context.Context, s bot.Session, m *user.Member, xp uint, fn func(uint, uint)) {
+	old := cc.XP
 	pastLevel := exp.Level(old)
 	s.Logger().Debug("adding xp", "user", m.DisplayName(), "old", old, "to add", xp)
-	c.CopaingXPs = append(c.CopaingXPs, CopaingXP{CopaingID: c.ID, XP: xp, GuildID: c.GuildID})
-	if err = c.Save(); err != nil {
-		s.Logger().Error("saving user", "error", err, "user", m.DisplayName(), "xp", xp, "guild", c.GuildID)
+	cc.XP += xp
+	cc.XPToAdd += xp
+	if err := cc.Save(ctx); err != nil {
+		s.Logger().Error("saving user in state", "error", err, "user", m.DisplayName(), "xp", xp, "guild", cc.GuildID)
 		return
 	}
 	newLevel := exp.Level(old + xp)
@@ -45,86 +40,27 @@ func (c *Copaing) AddXP(s bot.Session, m *user.Member, xp uint, fn func(uint, ui
 	}
 }
 
-func (c *Copaing) GetXP(logger *slog.Logger) (uint, error) {
-	cfg := config.GetGuildConfig(c.GuildID)
-	return c.GetXPForDays(logger, cfg.DaysXPRemains)
-}
-
-func (c *Copaing) GetXPForDays(logger *slog.Logger, n uint) (uint, error) {
+func (cc *CopaingCached) GetXPForDays(n uint) uint {
 	xp := uint(0)
-	rows, err := gokord.DB.
-		Model(&CopaingXP{}).
-		Where(
-			"created_at >= ? and guild_id = ? and copaing_id = ?",
-			exp.TimeStampNDaysBefore(n),
-			c.GuildID,
-			c.ID,
-		).
-		Rows()
-	if err != nil {
-		return 0, err
-	}
-	defer rows.Close()
-	for rows.Next() {
-		var cxp CopaingXP
-		err = gokord.DB.ScanRows(rows, &cxp)
-		if err != nil {
-			logger.Error("scanning rows", "error", err, "copaing", c.ID, "guild", c.GuildID)
-			continue
+	for _, v := range cc.XPs {
+		if v.Time <= time.Duration(n*24)*time.Hour {
+			xp += v.XP
 		}
-		xp += cxp.XP
 	}
-	return xp, nil
+	return xp + cc.XPToAdd
 }
 
 // GetBestXP returns n Copaing with the best XP within d days (d <= cfg.DaysXPRemain; d < 0 <=> d = cfg.DaysXPRemain)
-//
-// This function is slow
-func GetBestXP(logger *slog.Logger, guildId string, n uint, d int) ([]CopaingAccess, error) {
-	if d < 0 {
-		cfg := config.GetGuildConfig(guildId)
-		d = int(cfg.DaysXPRemains)
-	}
-	rows, err := gokord.DB.Model(&Copaing{}).Where("guild_id = ?", guildId).Rows()
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var l []*cXP
-	wg := sync.WaitGroup{}
-	for rows.Next() {
-		var c Copaing
-		err = gokord.DB.ScanRows(rows, &c)
-		if err != nil {
-			logger.Error("scanning rows", "error", err, "copaing", c.ID, "guild", c.GuildID)
-			continue
+func GetBestXP(ctx context.Context, guildId string, n uint, d int) ([]CopaingCached, error) {
+	ccs := GetState(ctx).Copaings(guildId)
+	if d > 0 {
+		for _, v := range ccs {
+			v.XP = v.GetXPForDays(n)
 		}
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			xp, err := c.GetXPForDays(logger, uint(d))
-			if err != nil {
-				logger.Error("fetching xp", "error", err, "copaing", c.ID, "guild", c.GuildID)
-				return
-			}
-			l = append(l, &cXP{Cxp: xp, Copaing: &c})
-		}()
 	}
-	wg.Wait()
-	slices.SortFunc(l, func(a, b *cXP) int {
+	slices.SortFunc(ccs, func(a, b CopaingCached) int {
 		// desc order
-		if a.Cxp < b.Cxp {
-			return 1
-		}
-		if a.Cxp > b.Cxp {
-			return -1
-		}
-		return 0
+		return int(b.XP) - int(a.XP)
 	})
-	m := min(len(l), int(n))
-	cs := make([]CopaingAccess, m)
-	for i, c := range l[:m] {
-		cs[i] = c
-	}
-	return cs, nil
+	return ccs[:min(len(ccs), int(n))], nil
 }