Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add get discord message logs api #755

Open
wants to merge 8 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions pkg/controller/discord/new.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ type IController interface {
ListDiscordResearchTopics(ctx context.Context, days, limit, offset int) ([]model.DiscordResearchTopic, int64, error)
UserOgifStats(ctx context.Context, discordID string, after time.Time) (OgifStats, error)
GetOgifLeaderboard(ctx context.Context, after time.Time, limit int) ([]model.OgifLeaderboardRecord, error)
ListDiscordChannelMessageLogs(ctx context.Context, channelID string, startTime *time.Time, endTime *time.Time) ([]model.DiscordTextMessageLog, error)
}

type controller struct {
Expand Down Expand Up @@ -335,3 +336,60 @@ func (c *controller) GetOgifLeaderboard(ctx context.Context, after time.Time, li

return leaderboard, nil
}

func (c *controller) ListDiscordChannelMessageLogs(ctx context.Context, channelID string, startTime *time.Time, endTime *time.Time) ([]model.DiscordTextMessageLog, error) {
threads, err := c.service.Discord.ListActiveThreadsByChannelID(c.config.Discord.IDs.DwarvesGuild, channelID)
if err != nil {
return nil, err
}
channelIDs := make([]string, 0)
channelIDs = append(channelIDs, channelID)
for _, thread := range threads {
channelIDs = append(channelIDs, thread.ID)
}

members, err := c.service.Discord.GetMembers()
if err != nil {
return nil, err
}

membersMap := make(map[string]string)
for _, mem := range members {
membersMap[fmt.Sprintf("<@%s>", mem.User.ID)] = fmt.Sprintf("@%s", mem.User.Username)
}

messageLogs := make([]model.DiscordTextMessageLog, 0)

for _, channnelID := range channelIDs {
messages, err := c.service.Discord.GetChannelMessagesInDateRange(channnelID, 100, startTime, endTime)
if err != nil {
return nil, err
}

for _, msg := range messages {
messageLogs = append(messageLogs, model.DiscordTextMessageLog{
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@namnhce @nnhuyhoang
this seems like a potential OOM, since on prod we only have 100MB-ish memory, make sure we dont OOM here

Copy link
Contributor Author

@nnhuyhoang nnhuyhoang Oct 8, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will test on dev with the same amount of messages, if cannot handle them , find another way

ID: msg.ID,
Content: msg.Content,
AuthorName: msg.Author.Username,
AuthorID: msg.Author.ID,
ChannelID: msg.ChannelID,
GuildID: msg.GuildID,
Timestamp: msg.Timestamp,
})
}
}

for _, msg := range messageLogs {
content := msg.Content
for id, mem := range membersMap {
if strings.Contains(content, id) {
content = strings.ReplaceAll(content, id, mem)
}
}
}

sort.Slice(messageLogs, func(i, j int) bool {
return messageLogs[i].Timestamp.Before(messageLogs[j].Timestamp)
})
return messageLogs, nil
}
49 changes: 49 additions & 0 deletions pkg/handler/discord/discord.go
Original file line number Diff line number Diff line change
Expand Up @@ -776,3 +776,52 @@

c.JSON(http.StatusOK, view.CreateResponse[any](nil, nil, nil, nil, "events swept successfully"))
}

// ListChannelMessageLogs godoc
// @Summary Get list of messages in channel and its thread
// @Description Get list of messages in channel and its thread
// @id ListChannelMessageLogs
// @Tags Discord
// @Accept json
// @Produce json
// @Param discord_channel_id path string true "Channel Discord ID"
// @Param startDate query string true "Start Date"
// @Param endDate query string true "End Date"
// @Success 200 {object} ListResearchTopicResponse
// @Failure 400 {object} ErrorResponse
// @Failure 404 {object} ErrorResponse
// @Failure 500 {object} ErrorResponse
// @Router /discords/{discord_channel_id}/message-logs [get]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should be updated

func (h *handler) ListChannelMessageLogs(c *gin.Context) {
var input = request.GetChannelMessagesInput{
DiscordChannelID: c.Param("discord_channel_id"),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@nnhuyhoang will this allow client call any discord_channel_id, if yes, gonna reject this for now

please follow up with me about this approach, need to validate/whitelist some channels only

}

if err := c.ShouldBindQuery(&input); err != nil {
c.JSON(http.StatusBadRequest, view.CreateResponse[any](nil, nil, err, input, "bind query failed"))
return
}

if err := input.Validate(); err != nil {
c.JSON(http.StatusBadRequest, view.CreateResponse[any](nil, nil, err, input, ""))
return
}

startDate := input.GetStartDate()
endDate := input.GetEndDate()

// maximum 3 month messages
threeMonths := time.Hour * 24 * 90
if endDate.Sub(*startDate) > threeMonths {
newEndDate := startDate.Add(threeMonths)
endDate = &newEndDate
}

messages, err := h.controller.Discord.ListDiscordChannelMessageLogs(c, input.DiscordChannelID, startDate, endDate)
if err != nil {
c.JSON(http.StatusInternalServerError, view.CreateResponse[any](nil, nil, err, nil, ""))
return
}
c.JSON(http.StatusOK, view.CreateResponse(messages, nil, nil, nil, ""))
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

define view


Check failure on line 826 in pkg/handler/discord/discord.go

View workflow job for this annotation

GitHub Actions / lint

unnecessary trailing newline (whitespace)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

failed linter

}
18 changes: 10 additions & 8 deletions pkg/handler/discord/errs/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@ package errs
import "errors"

var (
ErrEmptyReportView = errors.New("view is empty")
ErrEmptyChannelID = errors.New("channelID is empty")
ErrEmptyGuildID = errors.New("guildID is empty")
ErrEmptyCreatorID = errors.New("creatorID is empty")
ErrEmptyName = errors.New("name is empty")
ErrEmptyDate = errors.New("date is nil")
ErrEmptyID = errors.New("discord user id is nil")
ErrEmptyTopic = errors.New("topic is nil")
ErrEmptyReportView = errors.New("view is empty")
ErrEmptyChannelID = errors.New("channelID is empty")
ErrEmptyGuildID = errors.New("guildID is empty")
ErrEmptyCreatorID = errors.New("creatorID is empty")
ErrEmptyName = errors.New("name is empty")
ErrEmptyDate = errors.New("date is nil")
ErrEmptyID = errors.New("discord user id is nil")
ErrEmptyTopic = errors.New("topic is nil")
ErrInvalidDate = errors.New("date is invalid")
ErrInvalidTimeRange = errors.New("start time must be before end time")
)
1 change: 1 addition & 0 deletions pkg/handler/discord/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,5 @@ type IHandler interface {
UserOgifStats(c *gin.Context)
OgifLeaderboard(c *gin.Context)
SweepOgifEvent(c *gin.Context)
ListChannelMessageLogs(c *gin.Context)
}
48 changes: 48 additions & 0 deletions pkg/handler/discord/request/request.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,3 +97,51 @@ func (input DiscordEventSpeakerInput) Validate() error {
}
return nil
}

type GetChannelMessagesInput struct {
DiscordChannelID string
StartDate string `json:"startDate" form:"startDate"`
EndDate string `json:"endDate" form:"endDate"`
}

func (input GetChannelMessagesInput) Validate() error {
if len(input.DiscordChannelID) == 0 {
return errs.ErrEmptyChannelID
}
if (len(input.StartDate) == 0) || (len(input.EndDate) == 0) {
return errs.ErrEmptyDate
}

sTime, err := time.Parse("2006-01-02", input.StartDate)
if err != nil {
return errs.ErrInvalidDate
}
eTime, err := time.Parse("2006-01-02", input.EndDate)
if err != nil {
return errs.ErrInvalidDate
}

if sTime.After(eTime) {
return errs.ErrInvalidTimeRange
}

return nil
}

func (input GetChannelMessagesInput) GetStartDate() *time.Time {
date, err := time.Parse("2006-01-02", input.StartDate)
if err != nil {
return nil
}

return &date
}

func (input GetChannelMessagesInput) GetEndDate() *time.Time {
date, err := time.Parse("2006-01-02", input.EndDate)
if err != nil {
return nil
}

return &date
}
16 changes: 15 additions & 1 deletion pkg/model/discord_message.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
package model

import "github.com/bwmarrin/discordgo"
import (
"time"

"github.com/bwmarrin/discordgo"
)

type DiscordMessage struct {
AvatarURL string `json:"avatar_url"`
Expand Down Expand Up @@ -66,3 +70,13 @@ type OriginalDiscordMessage struct {
Author *discordgo.User
Roles []string
}

type DiscordTextMessageLog struct {
ID string `json:"id"`
Content string `json:"content"`
AuthorName string `json:"author_name"`
AuthorID string `json:"author_id"`
ChannelID string `json:"channel_id"`
GuildID string `json:"guild_id"`
Timestamp time.Time `json:"timestamp"`
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove json tag

}
3 changes: 2 additions & 1 deletion pkg/routes/v1.go
Original file line number Diff line number Diff line change
Expand Up @@ -381,13 +381,14 @@ func loadV1Routes(r *gin.Engine, h *handler.Handler, repo store.DBRepo, s *store
discordGroup.GET("/mma-scores", amw.WithAuth, pmw.WithPerm(model.PermissionEmployeesDiscordRead), h.Employee.ListWithMMAScore)
discordGroup.POST("/advance-salary", amw.WithAuth, pmw.WithPerm(model.PermissionEmployeesDiscordRead), h.Employee.SalaryAdvance)
discordGroup.POST("/check-advance-salary", amw.WithAuth, pmw.WithPerm(model.PermissionEmployeesDiscordRead), h.Employee.CheckSalaryAdvance)

discordGroup.GET("/salary-advance-report", amw.WithAuth, pmw.WithPerm(model.PermissionEmployeesDiscordRead), h.Employee.SalaryAdvanceReport)
discordGroup.GET("/:discord_id/earns/transactions", amw.WithAuth, pmw.WithPerm(model.PermissionEmployeesDiscordRead), h.Employee.GetEmployeeEarnTransactions)
discordGroup.GET("/:discord_id/earns/total", amw.WithAuth, pmw.WithPerm(model.PermissionEmployeesDiscordRead), h.Employee.GetEmployeeTotalEarn)
discordGroup.GET("/earns/total", amw.WithAuth, pmw.WithPerm(model.PermissionEmployeesDiscordRead), h.Employee.GetTotalEarn)
discordGroup.POST("/office-checkin", amw.WithAuth, pmw.WithPerm(model.PermissionTransferCheckinIcy), h.Employee.OfficeCheckIn)

discordGroup.GET("/channels/:discord_channel_id/message-logs", pmw.WithPerm(model.PermissionEmployeesDiscordRead), h.Discord.ListChannelMessageLogs)

discordGroup.GET("/icy-accounting", amw.WithAuth, pmw.WithPerm(model.PermissionEmployeesDiscordRead), h.Icy.Accounting)

scheduledEventGroup := discordGroup.Group("/scheduled-events")
Expand Down
28 changes: 28 additions & 0 deletions pkg/service/discord/discord.go
Original file line number Diff line number Diff line change
Expand Up @@ -826,3 +826,31 @@ func (d *discordClient) ListActiveThreadsByChannelID(guildID, channelID string)

return result, nil
}

func (d *discordClient) GetChannelMessagesInDateRange(channelID string, limit int, startDate, endDate *time.Time) ([]*discordgo.Message, error) {
before := ""
messages := make([]*discordgo.Message, 0)
for {
discordMessages, err := d.session.ChannelMessages(channelID, limit, before, "", "")
if err != nil {
return nil, err
}

if len(discordMessages) == 0 {
break
}

before = discordMessages[len(discordMessages)-1].ID

for _, msg := range discordMessages {
markedTime := msg.Timestamp
if markedTime.Before(*startDate) {
return messages, nil
}
if markedTime.After(*startDate) && markedTime.Before(*endDate) {
messages = append(messages, msg)
}
}
}
return messages, nil
}
3 changes: 3 additions & 0 deletions pkg/service/discord/service.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package discord

import (
"time"

"github.com/bwmarrin/discordgo"

"github.com/dwarvesf/fortress-api/pkg/model"
Expand Down Expand Up @@ -42,4 +44,5 @@ type IService interface {
SendDiscordMessageWithChannel(ses *discordgo.Session, msg *discordgo.Message, channelId string) error

ListActiveThreadsByChannelID(guildID, channelID string) ([]discordgo.Channel, error)
GetChannelMessagesInDateRange(channelID string, limit int, startDate, endDate *time.Time) ([]*discordgo.Message, error)
}
Loading